Pretrained_Bert_stopword_lemmatizer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "KuMWoh4eXWWf",
   "metadata": {
    "id": "KuMWoh4eXWWf"
   },
   "source": [
    "### <font color='violet'>Import libraries</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0529e0da",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "0529e0da",
    "outputId": "66c00fa4-15d3-4e9f-eeae-cbb10b83b280"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\diego\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\diego\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "C:\\Users\\diego\\anaconda3\\envs\\env1_cloned_tf\\lib\\site-packages\\tqdm\\auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "\n",
    "nltk.download('stopwords')\n",
    "nltk.download('wordnet')\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from tensorflow.keras import layers\n",
    "from sklearn.decomposition import PCA\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.image as mpimg\n",
    "import seaborn as sns\n",
    "import statistics\n",
    "from numpy import array\n",
    "from numpy import asarray\n",
    "from numpy import zeros\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from tensorflow.keras import layers\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "\n",
    "import contractions\n",
    "import re\n",
    "#!pip install transformers # install a library not present in Colab\n",
    "#!pip install contractions\n",
    "#!pip install spacy==2.2.3\n",
    "#!python -m spacy download en_core_web_sm\n",
    "#!pip install beautifulsoup4==4.9.1\n",
    "#!pip install textblob==0.15.3\n",
    "#!pip install contractions\n",
    "#!pip install transformers\n",
    "#! conda install text-hammer\n",
    "import transformers\n",
    "from transformers import BertTokenizer, TFBertForSequenceClassification"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "47af9ca8",
   "metadata": {},
   "source": [
    "### <font color='violet'>Preprocessing the datasets</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "a9838b56",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentences</th>\n",
       "      <th>emotion</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i didnt feel humiliated</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>i can go from feeling so hopeless to so damned...</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>im grabbing a minute to post i feel greedy wrong</td>\n",
       "      <td>anger</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>i am ever feeling nostalgic about the fireplac...</td>\n",
       "      <td>love</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>i am feeling grouchy</td>\n",
       "      <td>anger</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           sentences  emotion\n",
       "0                            i didnt feel humiliated  sadness\n",
       "1  i can go from feeling so hopeless to so damned...  sadness\n",
       "2   im grabbing a minute to post i feel greedy wrong    anger\n",
       "3  i am ever feeling nostalgic about the fireplac...     love\n",
       "4                               i am feeling grouchy    anger"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv(r'C:\\Users\\diego\\OneDrive\\Desktop\\DS\\Untitled Folder 1\\emotions_dataset\\train.txt', names=['sentences', 'emotion'], sep=';')\n",
    "val = pd.read_csv(r'C:\\Users\\diego\\OneDrive\\Desktop\\DS\\Untitled Folder 1\\emotions_dataset\\val.txt', names=['sentences', 'emotion'], sep=';')\n",
    "test = pd.read_csv(r'C:\\Users\\diego\\OneDrive\\Desktop\\DS\\Untitled Folder 1\\emotions_dataset\\test.txt', names=['sentences', 'emotion'], sep=';')\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "d2294b0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def Lemmatizer_stop_word(sentence):\n",
    "    stop_words = set(stopwords.words('english'))\n",
    "    lemmatizer = WordNetLemmatizer() #look at other Lemmatizers and stemmers\n",
    "    sentence = re.sub('[^A-z]', ' ', sentence)\n",
    "    negative = ['not', 'neither', 'nor', 'but', 'however',\n",
    "                'although', 'nonetheless', 'despite', 'except',\n",
    "                        'even though', 'yet']\n",
    "    stop_words = [z for z in stop_words if z not in negative]\n",
    "    preprocessed_tokens = [lemmatizer.lemmatize(contractions.fix(temp.lower())) for temp in sentence.split() if temp not in stop_words] #lemmatization\n",
    "    return ' '.join([x for x in preprocessed_tokens]).strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "cfe3d276",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package omw-1.4 to\n",
      "[nltk_data]     C:\\Users\\diego\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package omw-1.4 is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentences</th>\n",
       "      <th>emotion</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>did not feel humiliated</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>go feeling hopeless damned hopeful around some...</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>i am grabbing minute post feel greedy wrong</td>\n",
       "      <td>anger</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ever feeling nostalgic fireplace know still pr...</td>\n",
       "      <td>love</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>feeling grouchy</td>\n",
       "      <td>anger</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           sentences  emotion\n",
       "0                            did not feel humiliated  sadness\n",
       "1  go feeling hopeless damned hopeful around some...  sadness\n",
       "2        i am grabbing minute post feel greedy wrong    anger\n",
       "3  ever feeling nostalgic fireplace know still pr...     love\n",
       "4                                    feeling grouchy    anger"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('omw-1.4')\n",
    "train['sentences'] = train['sentences'].apply(lambda x: Lemmatizer_stop_word(x))\n",
    "val['sentences'] = val['sentences'].apply(lambda x: Lemmatizer_stop_word(x))\n",
    "test['sentences'] = test['sentences'].apply(lambda x: Lemmatizer_stop_word(x))\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a499c259",
   "metadata": {},
   "outputs": [],
   "source": [
    "#from imblearn.over_sampling import RandomOverSampler\n",
    "#ros = RandomOverSampler(random_state=0)\n",
    "#train_x, train_y = ros.fit_resample(np.array(train_data['text']).reshape(-1, 1), np.array(train_data['emotion']).reshape(-1, 1))\n",
    "#train = pd.DataFrame(list(zip([x[0] for x in train_x], train_y)), columns = ['text', 'emotion'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "O2S6U03yX5eN",
   "metadata": {
    "id": "O2S6U03yX5eN"
   },
   "source": [
    "### <font color='violet'>Tokenize the datasets</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "PJFUKQrpfs0J",
   "metadata": {
    "id": "PJFUKQrpfs0J"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some layers from the model checkpoint at bert-base-cased were not used when initializing TFBertModel: ['nsp___cls', 'mlm___cls']\n",
      "- This IS expected if you are initializing TFBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the layers of TFBertModel were initialized from the model checkpoint at bert-base-cased.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertModel for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "max_length=43\n",
    "from transformers import AutoTokenizer, TFBertModel\n",
    "tokenizer=AutoTokenizer.from_pretrained('bert-base-cased')\n",
    "bert=TFBertModel.from_pretrained('bert-base-cased')\n",
    "from tensorflow.keras.layers import Input, Dense\n",
    "x_train = tokenizer(\n",
    "    [x.split() for x in train['sentences']],\n",
    "    add_special_tokens=True,\n",
    "    max_length=max_length,\n",
    "    truncation=True,\n",
    "    padding=True, \n",
    "    return_tensors='tf',\n",
    "    return_token_type_ids = False,\n",
    "    return_attention_mask = True,\n",
    "    is_split_into_words=True,\n",
    "    verbose = True)\n",
    "\n",
    "\n",
    "x_val = tokenizer(\n",
    "    [x.split() for x in val['sentences']],\n",
    "    add_special_tokens=True,\n",
    "    max_length=max_length,\n",
    "    truncation=True,\n",
    "    padding=True, \n",
    "    return_tensors='tf',\n",
    "    return_token_type_ids = False,\n",
    "    return_attention_mask = True,\n",
    "    is_split_into_words=True,\n",
    "    verbose = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d717936c",
   "metadata": {},
   "outputs": [],
   "source": [
    "lb = LabelEncoder()\n",
    "labels_train=lb.fit(train.loc[:,'emotion'].to_list())\n",
    "labels_train=lb.transform(train.loc[:,'emotion'].to_list())\n",
    "labels_val=lb.transform(val.loc[:,'emotion'].to_list())\n",
    "labels_test=lb.transform(test.loc[:,'emotion'].to_list())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "qheaw4F0XEM3",
   "metadata": {
    "id": "qheaw4F0XEM3"
   },
   "source": [
    "### <font color='violet'>Train the model</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8e3cdc07",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "8e3cdc07",
    "outputId": "6497e6c1-ade0-48db-d741-9dc7607d9c3f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\diego\\AppData\\Roaming\\Python\\Python39\\site-packages\\tensorflow\\python\\ops\\array_ops.py:5043: calling gather (from tensorflow.python.ops.array_ops) with validate_indices is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "The `validate_indices` argument has no effect. Indices are always validated on CPU and never validated on GPU.\n",
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_ids (InputLayer)          [(None, 43)]         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "attention_mask (InputLayer)     [(None, 43)]         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "tf_bert_model_1 (TFBertModel)   TFBaseModelOutputWit 108310272   input_ids[0][0]                  \n",
      "                                                                 attention_mask[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d (GlobalMax (None, 768)          0           tf_bert_model_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 138)          106122      global_max_pooling1d[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "dropout_74 (Dropout)            (None, 138)          0           dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 28)           3892        dropout_74[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 6)            174         dense_1[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 108,420,460\n",
      "Trainable params: 108,420,460\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "None\n",
      "Epoch 1/3\n",
      "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model_1/bert/pooler/dense/kernel:0', 'tf_bert_model_1/bert/pooler/dense/bias:0'] when minimizing the loss.\n",
      "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model_1/bert/pooler/dense/kernel:0', 'tf_bert_model_1/bert/pooler/dense/bias:0'] when minimizing the loss.\n",
      "1334/1334 [==============================] - 959s 698ms/step - loss: 0.3806 - accuracy: 0.8639 - val_loss: 0.1853 - val_accuracy: 0.9315\n",
      "Epoch 2/3\n",
      "1334/1334 [==============================] - 924s 693ms/step - loss: 0.1362 - accuracy: 0.9416 - val_loss: 0.1656 - val_accuracy: 0.9305\n",
      "Epoch 3/3\n",
      "1334/1334 [==============================] - 933s 700ms/step - loss: 0.1113 - accuracy: 0.9482 - val_loss: 0.1550 - val_accuracy: 0.9375\n"
     ]
    }
   ],
   "source": [
    "tf.random.set_seed(79)\n",
    "#max_len = 43\n",
    "from tensorflow.keras.layers import Input, Dense\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from tensorflow.keras.initializers import TruncatedNormal\n",
    "from tensorflow.keras.losses import CategoricalCrossentropy\n",
    "from tensorflow.keras.metrics import CategoricalAccuracy\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "\n",
    "\n",
    "input_ids = Input(shape=(max_length,), dtype=tf.int32, name=\"input_ids\")\n",
    "input_mask = Input(shape=(max_length,), dtype=tf.int32, name=\"attention_mask\")\n",
    "# embeddings = dbert_model(input_ids,attention_mask = input_mask)[0]\n",
    "\n",
    "\n",
    "embeddings = bert(input_ids,attention_mask = input_mask)[0] #(0 is the last hidden states,1 is the pooler_output)\n",
    "x = tf.keras.layers.GlobalMaxPool1D()(embeddings)\n",
    "x = Dense(138, activation='elu',kernel_initializer='GlorotNormal')(x)\n",
    "x = tf.keras.layers.Dropout(0.1)(x)\n",
    "x = Dense(28,activation = 'elu',kernel_initializer='GlorotNormal')(x)\n",
    "\n",
    "output = Dense(6,activation = 'softmax')(x)\n",
    "    \n",
    "model = tf.keras.Model(inputs=[input_ids, input_mask], outputs=output)\n",
    "model.layers[2].trainable = True\n",
    "# for training bert lr must be small\n",
    "\n",
    "\n",
    "opt = Adam(\n",
    "    learning_rate=5e-05, # works well with BERTs\n",
    "    epsilon=1e-08,\n",
    "    decay=0.01,\n",
    "    clipnorm=1.0)\n",
    "#model = tf.keras.models.Model( bert_model.input , outputs )\n",
    "model.compile(optimizer=opt, loss='sparse_categorical_crossentropy', metrics=['accuracy']) \n",
    "\n",
    "#'sparse_categorical_crossentropy' for not one-hot encoded features\n",
    "# summarize the model\n",
    "print(model.summary())\n",
    "# fit the model\n",
    "early_stopping_cb=keras.callbacks.EarlyStopping(patience=2,restore_best_weights=True)\n",
    "#bert_model.trainable = False\n",
    "\n",
    "history = model.fit(\n",
    "    x ={'input_ids':x_train['input_ids'],'attention_mask':x_train['attention_mask']} ,\n",
    "    y =labels_train,\n",
    "    validation_data = (\n",
    "    {'input_ids':x_val['input_ids'],'attention_mask':x_val['attention_mask']}, labels_val\n",
    "    ),\n",
    "  epochs=3,\n",
    "    batch_size=12,callbacks=[early_stopping_cb]\n",
    ")\n",
    "\n",
    "#history=model.fit(padded_docs_train, labels_train, epochs=70,validation_data=(padded_docs_val, labels_val),callbacks=[early_stopping_cb],batch_size=20)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "iuMr1qvjYGwl",
   "metadata": {
    "id": "iuMr1qvjYGwl"
   },
   "source": [
    "### <font color='violet'> History plot </font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "dab9fef0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "63/63 [==============================] - 24s 382ms/step - loss: 0.1550 - accuracy: 0.9375\n",
      "Accuracy: 93.750000\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeMAAAEzCAYAAAACSWsXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAA3QklEQVR4nO3deXxU5aH/8c8zSzYSAmFJwr4IRkiICCKKImAFQlVaL1astZZW/dlae1tf9lpta71Xu2m1t4tXS73W2mqprdJrNQG1EqnWBVA0YTEioAYS9i1ASDLz/P6YSTKZTJIJmeRMhu/79ZrXnDnbPE8m53yf88yZc4y1FhEREXGOy+kCiIiInOoUxiIiIg5TGIuIiDhMYSwiIuIwhbGIiIjDFMYiIiIO6zCMjTGPGmN2G2PK25hujDG/NMZsMca8Z4w5K/bFFBERSVzRHBk/BsxvZ3oRMC74uAF4qOvFEhEROXV0GMbW2tXA/nZmWQg8bgPeAPoZY3JjVUAREZFEF4vvjIcCn4S8rgyOExERkSh4YrAOE2FcxGtsGmNuINCVTWpq6pThw4fH4O0D/H4/LldinI+musSnRKlLotQDVJd4lSh16Y56VFRU7LXWDgofH4swrgRCU3UYsDPSjNbapcBSgKlTp9q1a9fG4O0DSktLmTVrVszW5yTVJT4lSl0SpR6gusSrRKlLd9TDGPNRpPGxiPxngS8Gz6qeDhyy1lbFYL0iIiKnhA6PjI0xfwJmAQONMZXADwAvgLX2YaAYWABsAY4BS7qrsCIiIomowzC21l7VwXQL3BSzEomIiJxiev837CIiIr2cwlhERMRhCmMRERGHKYxFREQcpjAWERFxmMJYRETEYQpjERERhymMRUREHBaLa1OLiEhvYG3w4Q8+fM3D/sZh23paq+ntTfPT99Am+CgpbFpHy0Z4nOy0GK134u5dcN40SErr9o9GYSwi0WvcUbfamYXtyFtMD9sZt7msH/zhO8fwHXfLaQP3lMGGAx0v6/e1HUIR1tt659xegLW33tDp7a938qED8EF6DN6znbpYf4/8m5wF8E6PvFVkxhV8uJuHXW4wJvI04wKXq9W0tGPHA3/DHqAwFkk0J47A4So4shOOVMPhnXCkKvA4XMU5+yrhHW87odlO4MaZfIANsV6rCdl5h++4I01rnH6S01xuMF587lRI7R95ua68Z8zr0tayzcPvlpVReObkNpYND77wYDRtrret0GwVuDGyprSUWckZMVtfexTGIr2Frz4QrkeqA0F7uKo5ZINBy5EqqKtpvWxyJvTNhYwcDvc9ndQhw0J21pF2aI073PamRTrycEWxbIRHe8u2s9yadW9z9rRzolxvR9MaQyh2O/POeC/Gt+uz/sBRcuNz6LD1BY+ufT6sP9j48vmwTeP8Laf7g+Mah32h6wydHngcrfRwJNOH9deDL9CYa1GO0HGNw43rtCHTw8f5/Vi/L1COSONChiONC5QzbJzPh7X+ltOD4/ofOIh/2jRcaeqmjoq/thb3nj3UffJJYESLjal5uOVo07nhxvW0mNzJdbSxkYevxxw9iu/QocjLdkfdQoZblLCj929rHQ7tzHota+H4gVZHsM3DwSPco3sA23JZlxcycgNBmz0BTvtUMHSDj75DICMHkvo0LbKptJTsk9zpWxvYkbXccUe3M29rx219Fuobp0cIEJ8fbEPEnXnD5r0csh+2szNvZ8cdMj3qHXd4KHWwM2+xs298fxt5XNbhw2y9//7mcoZO97c9rq2/m5P6E7jRfbdwucDtDuxnXC6My9VqHG4XxrQzzhVsTDZOd7vBZVosY9werNcT+D/uAQkRxsffeYeB37+TD50uSIwMBiqcLkSMZAObGl/EopHQnQ2NDoYH1ddT4fV2bjkI6QKOcPKK9WEau4bDCxl69ObqA66+wdfBrk2XJ3hEBwYfsCPw6KBcA44fZ0tycsgOPDSI/C3DNmxcvOkH7IzFioI75OYdd/NOOtK48B13+DhcBuNyB8IidLrLjXG5MMlJwc+weT5fUhJJ2dktAyZ0Pe7GI/pI40KGG0MnfFywezfSuKZhVzCogu/fOC7wnm2NCy7jDvQsGJeLdevXM2XqVIw78P/bYro77O/SWKYW9XaFvV/IcA/aVlqKq0+fjmeMgYQI46SxYzn0pWvJy8treeAQ2qJp0bqJPN62Nb+NMO5k1xFFuT74YAvjThvbep0xq9vJredk6rb9o+2MGjmqxUwn/zeKUd3aqGdH69lRWcmAoUOD4/xQfxzqjjY/6o9h6441DVN3FHx1rdfn8oA3rfmR1Dic2jTOelIDO6DQ8saoboerdzEgNzfsaCFsx9zqCKKdnXDYuOawCBsXumMOG9diemjohO2sw3fcb61dy7Rzpre7M4+4Yw8N2cZgcNiHpaVMjmE3tZMaDhwgdeJEp4vRqyREGHsHD6Z2+nT6Jcg/8vHSUrISpC4bSksZ1JvqYi2cOBxy4lPz97PuflsZlPx+oAu5Zhe4fZBK4AGBAEnPDnYPjwl0D4d2FWcMCXQhJ/d17HtJgA9KSzmrN30m7fDt2EHymNFOF0OkyxIijEWi0lAHNdURzzBuMVx/tPWyKZmkuTKh71gYmxf8TjYnGLTB72fTBweO+EREOklhLL2ftXBsf+szjMOObDm2t/Wy7qTmo9bsfBg3N+Tkp5AToZLSAj9zSJAjShGJLwpjiW91x9o/w7jxt7SN382G6jOoOWiHTml9hnHGEEjLcrTLWEQEFMbiFL8PanZHuDBFWBdy7aHWy3r7NB+1Dp8eHB4S0m2cA+k54Enq+XqJiJwEhbHElrWBAA0eteZUrYLVa1sf2dbsan1pPuMOHrHmwIDTYPTM5iPY0O9nkzN0NCsiCUVhLNFrOBEM2TZOfGocrj/WtEgewPtASr/mo9bBE5quBtV0hnFGbqBbWSdAicgpSGEsgQs6HN8f4Qzjxi7k4PCxfa2XdSc3H7XmFsL4+S1OfHpj08dMv2hhj9z1RESkt1IYJ7q6o+2fYdx4pOuvD1vQBI5U++ZC5lAYNjXkDOOQbuPU/u12Gddur1cQi4h0QGHcW/ka4OjuCHfnCQvaExFOgEpKbw7WkedGvjBFeja4va2XFRGRmFMYxxtr8dTXwO5N7V+Y4uju1idAuTyBs4gzcmDQeBgzq/WFKfoGT4ASEZG4oTDuSQ0n2vm9bPPw+Q3H4bWwZVP7Nx+1Zk9sfYZx0wlQPXshdRER6TqFcSz4/YGrO7U68SksaI/vb72sJyXkwhRnQUYuW3Yf47TJ5zeHbEYueFN6vl4iItIjFMYdOVHTxhnGIRepOFId+QSo9MGBIM0cDsOnhRzNhpwEFeEEqMrSUk7Ln9VjVRQREWedumHsqw9eASrSGcYh4Vt3pPWyyX2b78gz6vzIF6ZIH6wToEREJCqJF8bWwvEDHV+YomY3LW8QS+AEqMa78QzKg7FzWl+YIiMXktMdqZqIiCSmxAjjHes4853b4d1jgSPchtrW86RmNR+15hS0PsM4IxfSBuoEKBER6XGJEcbuJMDA0KmRf8qTnqMToEREJG4lRhjnFLB+8o90r1kREemV1CcrIiLiMIWxiIiIwxTGIiIiDlMYi4iIOExhLCIi4jCFsYiIiMMUxiIiIg5TGIuIiDhMYSwiIuIwhbGIiIjDFMYiIiIOUxiLiIg4LKowNsbMN8a8b4zZYoz5ToTpmcaYvxtj3jXGbDDGLIl9UUVERBJTh2FsjHEDDwJFwATgKmPMhLDZbgI2WmsLgVnA/caYpBiXVUREJCFFc2Q8Ddhird1qra0DlgELw+axQIYxxgDpwH6gIaYlFRERSVDGWtv+DMYsAuZba68Lvr4GOMda+/WQeTKAZ4E8IAO40lr7fIR13QDcAJCdnT1l2bJlsaoHNTU1pKenx2x9TlJd4lOi1CVR6gGqS7xKlLp0Rz1mz569zlo7NXy8J4plTYRx4Qk+D1gPzAHGAi8aY/5prT3cYiFrlwJLAaZOnWpnzZoVxdtHp7S0lFiuz0mqS3xKlLokSj1AdYlXiVKXnqxHNN3UlcDwkNfDgJ1h8ywBnrEBW4BtBI6SRUREpAPRhPEaYJwxZnTwpKzFBLqkQ30MXARgjMkGTge2xrKgIiIiiarDbmprbYMx5uvASsANPGqt3WCMuTE4/WHgbuAxY0wZgW7t26y1e7ux3CIiIgkjmu+MsdYWA8Vh4x4OGd4JzI1t0URERE4NugKXiIiIwxTGIiIiDlMYi4iIOExhLCIi4jCFsYiIiMMUxiIiIg5TGIuIiDhMYSwiIuIwhbGIiIjDFMYiIiIOUxiLiIg4TGEsIiLiMIWxiIiIwxTGIiIiDlMYi4iIOExhLCIi4jCFsYiIiMMUxiIiIg5TGIuIiDhMYSwiIuIwhbGIiIjDFMYiIiIOUxiLiIg4TGEsIiLiMIWxiIiIwxTGIiIiDlMYi4iIOExhLCIi4jCFsYiIiMMUxiIiIg5TGIuIiDhMYSwiIuIwhbGIiIjDFMYiIiIOUxiLiIg4TGEsIiLiMIWxiIiIwxTGIiIiDlMYi4iIOExhLCIi4jCFsYiIiMMUxiIiIg5TGIuIiDjM43QBRESk6+rr66msrKS2ttbpopCZmcmmTZucLkaXdaUeKSkpDBs2DK/XG9X8CmMRkQRQWVlJRkYGo0aNwhjjaFmOHDlCRkaGo2WIhZOth7WWffv2UVlZyejRo6NaJqpuamPMfGPM+8aYLcaY77QxzyxjzHpjzAZjzCudKLeIiHRRbW0tAwYMcDyIBYwxDBgwoFO9FB0eGRtj3MCDwMVAJbDGGPOstXZjyDz9gP8B5ltrPzbGDO5s4UVEpGsUxPGjs59FNEfG04At1tqt1to6YBmwMGyezwPPWGs/BrDW7u5UKURERE5h0YTxUOCTkNeVwXGhxgP9jTGlxph1xpgvxqqAIiLSO6SnpztdhF4rmhO4Ih1r2wjrmQJcBKQCrxtj3rDWVrRYkTE3ADcAZGdnU1pa2ukCt6Wmpiam63OS6hKfEqUuiVIPUF1CZWZmcuTIkdgV6CQdOXIEn88XF2Xpqq7Wo7a2NvrP1Frb7gM4F1gZ8vp24Paweb4D3BXy+n+BK9pb75QpU2wsrVq1Kqbrc5LqEp8SpS6JUg9rVZdQGzdujE1BuqBPnz7WWmsPHTpkb731Vjtx4kSbn59vly1bZq21dufOnfaCCy6whYWFduLEiXb16tW2oaHBXnvttU3zPvDAA05WoYXDhw93aflInwmw1kbIxGiOjNcA44wxo4EdwGIC3xGH+j/g18YYD5AEnAP8PLrmgIiIxNJ//n0DG3cejuk6Jwzpyw8unRjVvM8++yzr16/n3XffZe/evZx99tnMnDmTJ598knnz5vHd734Xn8/HsWPHWL9+PTt27KC8vByAgwcPxrTcvUWHYWytbTDGfB1YCbiBR621G4wxNwanP2yt3WSMWQG8B/iBR6y15d1ZcBERiU+vv/46V111FW63m+zsbC688ELWrFnD2WefzZe//GXq6+v5zGc+w5lnnsmYMWPYunUrN998M5/+9KeZO3eu08V3RFQX/bDWFgPFYeMeDnt9H3Bf7IomIiInI9oj2O4S6I1tbebMmaxevZrnn3+ea665hm9/+9t88Ytf5N1332XlypU8+OCDPPXUUzz66KM9XGLn6drUIiISUzNmzODPf/4zPp+PPXv2sHr1aqZNm8ZHH33E4MGDuf766/nKV77C22+/zd69e/H7/fzbv/0bd999N2+//bbTxXeELocpIiIxdemll7J+/XoKCwsxxnDvvfeSk5PD73//e+677z68Xi/p6ek8/vjj7NixgyVLluD3+wH48Y9/7HDpnaEwFhGRmKipqQECV5+67777uO++lt9cXnvttVx77bWtljtVj4ZDqZtaRETEYQpjERERhymMRUREHKYwFhERcZjCWERExGEKYxEREYcpjEVERBymMBYRkV6joaHB6SJ0C4WxiIjExGc+8xmmTJnCtGnTWLp0KQArVqzgrLPOorCwkIsuuggIXBxkyZIlFBQUMGnSJJ5++mkA0tPTm9b117/+lS996UsAfOlLX+KWW25h9uzZ3Hbbbbz11lucd955TJ48mfPOO4/3338fCNx/+NZbb21a769+9Sv+8Y9/8NnPfrZpvS+++CKXX355T/w5OkVX4BIRSTQl34HqstiuM6cAin7S7iyPPvooWVlZ7N69mzlz5rBw4UKuv/56Vq9ezejRo9m/fz8Ad999N5mZmZSVBcp44MCBDt++oqKCl156CbfbzeHDh1m9ejUej4eXXnqJO+64g6effpqlS5eybds23nnnHTweD/v376d///7cdNNN7Nmzh0GDBvG73/2OJUuWdP3vEWMKYxERiYlf/vKXLF++HL/fzyeffMLSpUuZOXMmo0ePBiArKwuAl156iWXLljUt179//w7XfcUVV+B2uwE4dOgQ1157LR988AHGGOrr65vWe+ONN+LxeFq83zXXXMMf//hHlixZwuuvv87jjz8eu0rHiMJYRCTRdHAE2x1KS0t56aWXeP311/H5fFx66aUUFhY2dSGHstZijGk1PnRcbW1ti2l9+vRpGv7+97/P7NmzWb58Odu3b2fWrFntrnfJkiVceumlpKSkcMUVVzSFdTzRd8YiItJlhw4don///qSlpVFRUcEbb7zBiRMneOWVV9i2bRtAUzf13Llz+fWvf920bGM3dXZ2Nps2bcLv97N8+fJ232vo0KEAPPbYY03j586dy8MPP9x0klfj+w0ZMoQhQ4Zwzz33NH0PHW8UxiIi0mXz58+noaGBSZMmcc899zB9+nQGDRrE0qVLufzyyyksLOTKK68E4Hvf+x4HDhwgPz+fwsJCVq1aBcBPfvITLrnkEubMmUNubm6b7/Uf//Ef3H777cyYMQOfz9c0/rrrrmPEiBFMmjSJwsJCnnzyyaZpV199NcOHD2fChAnd9Bfomvg7VhcRkV4nOTmZkpISAI4cOUJGRkbTtKKiohbzpqen8/vf/77VOhYtWsSiRYtajQ89+gU499xzqaioaHp99913A+DxeHjggQd44IEHWq3j1Vdf5frrr4++Qj1MYSwiIgltypQp9OnTh/vvv9/porRJYSwiIglt3bp1ThehQ/rOWERExGEKYxEREYcpjEVERBymMBYREXGYwlhERMRhCmMREelxoXdoCrd9+3by8/N7sDTOUxiLiIg4TL8zFhFJMD9966ds3r85puvMy8rjtmm3tTn9tttuY+TIkXzta18D4K677sIYw+rVqzlw4AD19fXcc889LFy4sFPvW1tby1e/+lXWrl3bdIWt2bNns2HDBpYsWUJdXR1+v5+nn36aIUOG8LnPfY7Kykp8Ph/f//73my7BGe8UxiIi0mWLFy/mm9/8ZlMYP/XUU6xYsYJvfetb9O3bl7179zJ9+nQuu+yyiHdWasuDDz4IQFlZGZs3b2bu3LlUVFTw8MMP8+///u9cffXV1NXV4fP5KC4uZsiQITz//PNA4IYSvYXCWEQkwbR3BNtdJk+ezO7du9m5cyfbt2+nf//+5Obm8q1vfYvVq1fjcrnYsWMHu3btIicnJ+r1vvrqq9x8880A5OXlMXLkSCoqKjj33HP54Q9/SGVlJZdffjnjxo2joKCAW2+9ldtuu41LLrmECy64oLuqG3P6zlhERGJi0aJF/PWvf+WZZ55h8eLFPPHEE+zZs4d169axfv16srOzW92nuCPW2ojjP//5z/Pss8+SmprKvHnzePnllxk/fjzr1q2joKCA22+/nf/6r/+KRbV6hI6MRUQkJhYvXsz111/P7t27+ec//8lTTz3F4MGD8Xq9rFq1io8++qjT65w5cyZPPPEEc+bMoaKigo8//pjTTz+drVu3MmbMGL7xjW+wdetW3nvvPfLy8sjKyuILX/gC6enpre72FM8UxiIiEhMTJ07kyJEjDBkyhNzcXK6++mouvfRSpk6dyplnnkleXl6n1/m1r32NG2+8kYKCAjweD4899hjJycn8+c9/5o9//CNer5ecnBzuvPNO1qxZw7e//W1cLhder5eHHnqoG2rZPRTGIiISM2VlZRw5cgSAgQMH8vrrr0ecr6amps11jBo1ivLycgBSUlIiHuHefvvt3H777S3GzZs3j3nz5p1kyZ2l74xFREQcpiNjERFxRFlZGddcc02LccnJybz55psOlcg5CmMREXFEQUEB69evd7oYcUHd1CIiIg5TGIuIiDhMYSwiIuIwhbGIiIjDFMYiItLj2ruf8alIYSwiIqeshoYGp4sA6KdNIiIJp/pHP+LEptjezzj5jDxy7rijzemxvJ9xTU0NCxcujLjc448/zs9+9jOMMUyaNIk//OEP7Nq1ixtvvJGtW7cC8NBDDzFkyBAuueSSpit5/exnP6Ompoa77rqLWbNmcd555/Haa69x2WWXMX78eO655x7q6uoYMGAATzzxBNnZ2dTU1PCNb3yDtWvXYozhBz/4AQcPHqS8vJyf//znAPz2t79l06ZNPPDAA136+yqMRUSky2J5P+OUlBSWL1/earmNGzfywx/+kNdee42BAweyf/9+AL7xjW9w4YUXsnz5cnw+HzU1NRw4cKDd9zh48CCvvPIKAAcOHOCNN97AGMMjjzzCvffey/3338+9995LZmYmZWVlTfMlJSUxadIk7r33XrxeL7/73e/4zW9+09U/X3RhbIyZD/wCcAOPWGt/0sZ8ZwNvAFdaa//a5dKJiEintXcE211ieT9jay133HFHq+VefvllFi1axMCBAwHIysoC4OWXX+bxxx8HwO12k5mZ2WEYX3nllU3DlZWVXHnllVRVVVFXV8fo0aMBKC0t5amnnmqar3///gDMmTOH5557jjPOOIP6+noKCgo6+ddqrcMwNsa4gQeBi4FKYI0x5llr7cYI8/0UWNnlUomISK/TeD/jjz/+uNX9jL1eL6NGjYrqfsZtLWet7fCoupHH48Hv9ze9Dn/fPn36NA3ffPPN3HLLLVx22WWUlpZy1113AbT5ftdddx0/+tGPyMvLY8mSJVGVpyPRnMA1Ddhird1qra0DlgGROv1vBp4GdsekZCIi0qssXryYZcuW8be//Y1FixZx6NChk7qfcVvLXXTRRTz11FPs27cPoKmb+qKLLmq6XaLP5+Pw4cNkZ2eze/du9u3bx4kTJ3juuefafb+hQ4cC8Pvf/75p/Jw5c/j1r3/d9LrxaPucc87hk08+4cknn+Sqq66K9s/TrmjCeCjwScjryuC4JsaYocBngYdjUioREel1It3PeO3atUydOpUnnngi6vsZt7XcxIkT+e53v8uFF15IYWEht9xyCwC/+MUvWLVqFQUFBUyZMoUNGzbg9Xq58847Oeecc7jkkkvafe+77rqLK664ggsuuKCpCxzg29/+NgcOHCA/P5/CwkJWrVrVNO1zn/scM2bMaOq67ipjrW1/BmOuAOZZa68Lvr4GmGatvTlknr8A91tr3zDGPAY8F+k7Y2PMDcANANnZ2VOWLVsWk0pA4Oy7RPndmuoSnxKlLolSD1BdQmVmZnLaaafFsEQnz+fz4Xa7nS5Gl7VXjyuuuIKbbrqJWbNmtbn8li1bOHToUItxs2fPXmetndpqZmttuw/gXGBlyOvbgdvD5tkGbA8+agh0VX+mvfVOmTLFxtKqVatiuj4nqS7xKVHqkij1sFZ1CbVx48bYFCQGDh8+7HQRYiJSPQ4cOGDHjRtnFy1a1OHykT4TYK2NkInRnE29BhhnjBkN7AAWA58PC/TRjcMhR8Z/i2LdIiJyiuqN9zPu168fFRUVMV9vh2FsrW0wxnydwFnSbuBRa+0GY8yNwen6nlhEJA7YTpxtHA8S+X7GtoOvgMNF9Ttja20xUBw2LmIIW2u/1KkSxEDNiQZe3F7P6YeOk5uZ2tNvLyLiuJSUFPbt28eAAQN6VSAnImst+/btIyUlJeplEuIKXK9/uI8nNtfxxI9f5qwR/VhQkMv8/ByG9U9zumgiIj1i2LBhVFZWsmfPHqeLQm1tbaeCKF51pR4pKSkMGzYs6vkTIowvnpDNj85PZX/acErKq7nn+U3c8/wmCodlMj8/lwUFOYwc0KfjFYmI9FJer7fpylFOKy0tZfLkyU4Xo8t6sh4JEcYAQ9JdfH7WOL4+Zxwf7TtKSXk1JWVV/HTFZn66YjMTcvuyoCCHooJcxg5KjJ9CiIhIYkiYMA41ckAfbrxwLDdeOJbKA8dYUV5NcVkVP3uhgp+9UMHp2RnMz89hQUEu47PT9f2KiIg4KiHDONSw/mlcd8EYrrtgDFWHjrOyvJri8mp++fIH/OIfHzBmUB8W5OdSVJDDhNy+CmYREelxCR/GoXIzU/nSjNF8acZodh+pZeWGXawor+J/Srfw61VbGJGVRlFBDgvyc5k0LFPBLCIiPeKUCuNQgzNSuGb6SK6ZPpJ9NSd4ceMuisur+d9/buM3r2xlaL9UivJzKCrIYfLw/rhcCmYREekep2wYhxqQnsziaSNYPG0EB4/V8eLGXawor+bx1z/ikVe3kd03maL8XIryc5g6Kgu3gllERGJIYRymX1oSV0wdzhVTh3O4tp6XN+2muKyKP731MY/9azsD05OZNzGbBQW5nDM6C487mhtfiYiItE1h3I6+KV4+M3kon5k8lKMnGlj1/m5Kyqp55u0dPPHmx/RP8zJvYuDnUueNHYBXwSwiIidBYRylPskeLpk0hEsmDeF4nY9XKnZTXFbNc+9VsWzNJ/RN8XDxhBwWFORw/riBJHt6/+3DRESkZyiMT0Jqkpv5+bnMz8+ltt7Hqx/spbi8ihc2VvP025VkJHu46IzBFBXkcuH4QaR4FcwiItI2hXEXpXjdfGpCNp+akE1dg5/XPtzLirJqVm6s5m/rd5KW5GZ23mAW5OcyO28QaUn6k4uISEtKhhhK8riYffpgZp8+mHt8+by5dT/F5VWsLK/m+feqSPG6mDV+MEUFOczJG0xGitfpIouISBxQGHcTr9vF+eMGcv64gdy9MJ+3tu2npLyKFeXVrNhQTZLbxczxAynKz+VTE7LJTFUwi4icqhTGPcDtMpw7dgDnjh3AXZdO5O2PD1BcVk1JeRUvbdqN122YcdpAFuTncvGEbKeLKyIiPUxh3MNcLsPUUVlMHZXF9z59Bu9WHgzcyKK8iv94+j3cyw15/Q07Uj9i7oQcBmUkO11kERHpZgpjB7lchskj+jN5RH++U5THhp2HKS6r4um3tvLd5eV8/2/lTBudRVF+LvPzc8ju2/tv1i0iIq0pjOOEMYb8oZnkD83k7OQqcs+YEujKLqviB89u4K6/b2DKiP4UFQSCeWi/VKeLLCIiMaIwjkPGGPJy+pKX05dbLh7PB7uOUFJeTUl5NXc/t5G7n9tI4fB+LMjPoSg/lxED0pwusoiIdIHCuBcYl53BuOwMvnHROLbtPUpJeRUlZdX8uGQzPy7ZTP7Qvk03shgzKN3p4oqISCcpjHuZ0QP78LVZp/G1Wafxyf5jTSd/3bfyfe5b+T55ORkU5eeyoCCHcdkZThdXRESioDDuxYZnpXH9zDFcP3MMOw8eZ0V54OdS//2PCn7+UgWnDU4P3JM5P5czcjMwRrd+FBGJRwrjBDGkXypfPn80Xz5/NLsP17JyQzXFZdU8uGoLv3p5C6MGpFFUkMuC/Fzyh/ZVMIuIxBGFcQIa3DeFa84dxTXnjmJvzQle2LCLkvIqlq7eykOlHzKsf2rgiLkglzOH9cPlUjCLiDhJYZzgBqYn8/lzRvD5c0Zw4GgdL27aRUlZFY/9azu//ec2cjNTmJ+fw4KCXKaM6K9gFhFxgML4FNK/TxKfmzqcz00dzqHj9fxj0y6Ky6p54s2P+d1r2xmckcy8iTkUFeQwbVQWHrfL6SKLiJwSFManqMxUL5efNYzLzxpGzYkGXt68m5KyKv6y7hP+8MZHDOiTxNyJOSwoyGH6mAF4FcwiIt1GYSykJ3u4rHAIlxUO4VhdA6Xv76G4rIr/W7+DP731Mf3SvFx8RjYLCnKZcdpAkjwKZhGRWFIYSwtpSR4WFOSyoCCX2nofqyv2UFJezYryav6yrpKMFA8Xn5HN/PwcZo4fRIrX7XSRRUR6PYWxtCnF62buxBzmTszhRIOP17bspaSsmhc27uKZd3bQJ8nNnDOyWZCfw6zTB5OapGAWETkZCmOJSrLHzZy8bObkZfMjn5/XP9xHSXkVKzfs4u/v7iTV62Z23iDm5+cyJ28w6cn61xIRiZb2mNJpXreLmeMHMXP8IO5e6Oet7fspKatmRfBCI0keFxeOH8SCghyS663TxRURiXsKY+kSj9vFeWMHct7Ygdx12UTWfXSA4rIqVpRX8+LGXbgN/KXyLYoKcpk7IZt+aUlOF1lEJO4ojCVm3C7DtNFZTBudxZ2XTGB95UGWFr9F2a4aVr3/Hne4DOeOHcCCYDAPSE92usgiInFBYSzdwuUynDWiP4vzknnowgsp23GI4rLAjSxuf6aM7y4v45zRA1hQkMO8iTkM7pvidJFFRByjMJZuZ4xh0rB+TBrWj9vmn86mqiOUlFfxfFkV3/+/Ddz57AbOHplFUUEO8/NzyM1MdbrIIiI9SmEsPcoYw4QhfZkwpC+3XDyeD3bXNH3H/J9/38h//n0jk0f0Y0F+LvPzcxieleZ0kUVEup3CWBxjjGF8dgbjszP45qfG8+GeGlaUV1NcVsUPizfxw+JNTBqWSVF+LkX5OYwa2MfpIouIdAuFscSNsYPSuWn2adw0+zQ+3neMkvIqisur+emKzfx0xWbOyO3LguCtH08bnO50cUVEYkZhLHFpxIA0/t+FY/l/F46l8sAxVpRXU1Jezf0vVnD/ixWMz05nfn4uCwpyOD07A2N060cR6b0UxhL3hvVP47oLxnDdBWOoPlTLyg2BruxfvfwBv/zHB4wZ2IeighyK8nOZOKSvgllEeh2FsfQqOZkpXHveKK49bxS7j9TywoZdlJRX8fArW3lw1YcMz0plQX4uRQW5FA7LVDCLSK+gMJZea3BGCl+YPpIvTB/J/qN1vLgxcDnO/311G79ZvZWh/VKZn59DUX4OZ43oj8ulYBaR+KQwloSQ1SeJK88ewZVnj+DQsXpe3LSLFeVV/OH1j/jfV7eR3TeZ+RMDJ3+dPSoLt4JZROKIwlgSTmaal0VThrFoyjCO1Nbz8ubdFJdVsWzNJ/z+9Y8YmJ7E3Ik5LMjPZfqYLDxul9NFFpFTXFRhbIyZD/wCcAOPWGt/Ejb9auC24Msa4KvW2ndjWVCRk5GR4mXhmUNZeOZQjp5oYNX7uykpq2b52zt48s2P6Z/mZe6EHIoKcjhv7ECSPApmEel5HYaxMcYNPAhcDFQCa4wxz1prN4bMtg240Fp7wBhTBCwFzumOAoucrD7JHi6ZNIRLJg3heJ2PVyr2NF2W889rP6FviodPTchmQX4u548bSIrX7XSRReQUEc2R8TRgi7V2K4AxZhmwEGgKY2vtv0LmfwMYFstCisRaapKb+fmBa2HX1vt49YO9lJRX8+LGap55ewfpyR4uOmMwRfm5zDp9kIJZRLqVsbb9m78bYxYB86211wVfXwOcY639ehvz3wrkNc4fNu0G4AaA7OzsKcuWLeti8ZvV1NSQnp4YV2VSXZzT4Lds3Odj7S4fb+9qoKYekt0waZCbgswGpg3vQ4qnd5/81ds+k/aoLvEpUerSHfWYPXv2Omvt1PDx0RwZR9rzRExwY8xs4CvA+ZGmW2uXEujCZurUqXbWrFlRvH10SktLieX6nKS6OOtTwecGn583t+2nuKyKlRuqWVPt44kPa5l1+iAWFOQyJ28wGSleR8t6MnrjZ9IW1SU+JUpderIe0YRxJTA85PUwYGf4TMaYScAjQJG1dl9siifiHI/bxYzTBjLjtIH818J8fvu3l6lyZ7NiQzUrN+wiye3ignEDKSrI5eIzsslM633BLCLxIZowXgOMM8aMBnYAi4HPh85gjBkBPANcY62tiHkpO7Dv+D7eOfoOfAJJriS8bi9el5ckd1LT66bnxnEuL26XvgeU6LhdhrwsNzfOyucHl07knU8OUFxWTUlZFf/YvBuPyzDjtIEsKMjh4gk5ZPVJcrrIItKLdBjG1toGY8zXgZUEftr0qLV2gzHmxuD0h4E7gQHA/wQvP9gQqU+8u3xw8AMe3fsovNy55dzGTZI7EMxN4R3hdWN4h4d5kjsk6MOWD32O1ECINF/js8Q3l8swZWQWU0Zm8b1Pn8G7lYcoKa+ipKya254u447l5Uwfk0VRfi7zJuYwKCPZ6SKLSJyL6nfG1tpioDhs3MMhw9cBrU7Y6imTBk7i9tzbKTyrkDp/HfW+eur8ddT56ppe1/vrA68bx/nrA/OFvK7zNS8bOv/hhsOtlm/wN7R4HUsuXCQ/kdw6vBsDvY3XbTUsQnsFOpo/YsMj2ODQdZ5bM8Zw5vB+nDm8H9+Zn8eGnYebgvl7fyvn+/9XzrRRWRTl5zA/P5eczBSniywicSghrsCV5k1jSNIQJg6c6Mj7W2tp8Dc0B3awIdAU8GHPrRoIIfPX++r5YNsH5A7Lbb1c43Bw+WP1x5peNzYkWrz218e0nh6Xp0U4N4V5yOvwXoD9e/dT+q/SwLJR9Aq02YvQTsMjXhoJxhjyh2aSPzSTW+eeTsWuGorLqigpr+Kuv2/krr9vZMrI/hQF78k8tF+q00UWkTiREGHsNGNMICzcXtK8aV1eX+nBUmadPavL62lsJDT1EoSFeTS9AvX+CL0KYQ2JpuWCzzX1NU3TD9ceZnvl9hbzN/gbuly3UC0CvY1egGh6ETo632Dz8c2kVKV02BBJciXhcXk4PSeD03My+NbF49my+wglZdUUl1dzz/ObuOf5TRQOy6SoIJei/BxGDugT07+JiPQuCuMEFtpI6ON1Zmcf6acBfutv7uYP6xWIplchUi9ApIZDaMPjRMMJavw1LRoi4b0KDTaKRsIL0dc9YmiPTGLSCDdHa2HHMcsvyuEX5W4yklMYmpnB8P59yUpLbdWL0CL0I4yL5vwEr9uLx2iTF4lH2jKlx7mMqylU4onf+lv3CoQ0EN5Y+wYFhQURewMizd/u1xHpwV6EE7XsO3qMg7V7qDiwg4pDDXjcfjxuP8bVQIO/Hp/1xayOBoMbN94nAsHsdrlxGVfLYZcHl3HhNu6m4Tbn7aZ1uI27+Tk4HGneLbVb6Le7X8T52xwXNuwyrrj5qkNOXQpjkSCXcZHsTibZHfns513Ju5ia030/Eqg6dJwV5dWUlFWz5qP9WAtjB/Vhfv5gLp4wiNGDk5oCv63zDaLpWfhw+4cMHTa0qYfCZ30thn3Wh8/va3O43l/Pcd9xfP7gcraheTjKddjI1w06OSVdX8XJhHinhqOYt/JgJRvXb2x3HR4TbOSEDUdaX1ODqI3lPC5PU2MkfLixIaVGSs9RGIvEidzMVJbMGM2SGaPZfbiWlRuqKSmv5qHSbTy4ahsjB6RRlJ/LgoIcCobmnPSOsvRQbM5J6Aq/9TcFc2igdxTi4cNvv/M2+ZPyO71c43CDbQiUJYr3Dp030nCDv4FaW4vf7z+pdfj8Puy7MWykxEBbYd3R8LGaYzxS/EibjYJo19eqd6UT5Wizd6Wd6eHDR31H8Vs/LtP9d3NTGIvEocF9U7jm3FFcc+4o9tWc4IWNuyguq+K3/9zKw698yNB+qSwoCPxcavLwfrhcvesIxmVcuIwLr6trVy07mnqUGUNnxKhUziotLWXmhTObeipCGwgdNQTCGwU+68Pv97c/PYrhSI2UaJbbc2wPqZ5UfNbXel1RriN82Ckz62eSkZTR7e+jMBaJcwPSk7lq2giumjaCg8fqeHHjLkrKq3nsX9v57T+3kdM3hfn5OSwoyGXKyP64e1kwS7PGRgoQuMRSLxXrazpba1v1NnT6a5ZIvR8d9IZsrthMirtnrg2gMBbpRfqlJXHF1OFcMXU4h47X8/LmXRSXVfPkWx/z2L+2MygjmXkTA/dknjY6C4+7+7vXRLqbMQaP8eDB06ONlNLqUrzunrnmvMJYpJfKTPXy2cnD+OzkYdScaGDV5t2UlFfx13WV/PGNj8nqk8S8idkU5edy7tgBeBXMInFLYSySANKTPVxaOIRLC4dwrK6BV97fQ3F5Nc+u38mf3vqEzFQvF0/IZkFBDj5/fJ0kJCIKY5GEk5bkCVzZqyCX2nof//xgLyVlVawsr+av6yrxGEh/9QVSPG5SvC6SQ56Tw15HfnaR4g3Mm+Jp+Ry+rhSvmxSPG6/b6GcyIu1QGIsksBSvm4snZHPxhGxONPj415Z9/OWV9QzMGUJtvY8TDf4WzzUnGthbU8eJCNMaunBEbQytgjtSkLc3PcXjItnbsmGweZ+PjI/2R2wEJHtcJLl1QQ/pHRTGIqeIZI+b2XmDMdVJzJqV3+nlG3x+TjT4W4V0e8+RQv1EvZ/ahubnFo2A4PgTDT5qg8/1vg4aAWteb3OSMTQfybfznNzB9NDntnoNQnsG1AiQzlIYi0hUPG4XHreLPj18e2af37YI59DnN9esIy9/EifqfdQGw782pBHQ+DpSI+BEvZ+aEw3sq2l+fSJkeoeNgHY0NgI67vJvDvG9u07wxvHNLaZH0zgIbUwke9QI6K0UxiIS19wuQ1qSh7QIlzI/+KGbC8cP6pb3bWwERArxaHoEmhoHERoRx+oa2H80dLyfo7UNvFK5jTpf1y5wEe0RfniPQEeNhvYbE2oEdJXCWEQkgvYaAd2h8UIZPr+lLsqvAqL6WiDk+VhdAweOhY5v7g2IRSOgMdhtQx393n6lw3MEomo0tLuOxGkEKIxFROKI22VITXKTmtSzl+Dy+23wnICWR/Enc37ARzt20i8rPaTR4OfgsfqIy9U1dK0RkOSJfHJfU1d/p08UbD4JcNshH+f7/D1y8RyFsYiI4IphI6C0dD+zZk2Jal6/31Lnaw7tts4PaO85/KS/0OdDx+sjNi6ibQT82zwffRXGIiKSyFwuQ4or0GXdk0IbAW0d8a99513SeqhcCmMRETnlRNMI8O3w9Nj13XWxWhEREYcpjEVERBymMBYREXGYwlhERMRhCmMRERGHKYxFREQcpjAWERFxmMJYRETEYQpjERERhymMRUREHKYwFhERcZjCWERExGEKYxEREYcpjEVERBymMBYREXGYwlhERMRhCmMRERGHKYxFREQcpjAWERFxmMJYRETEYQpjERERhymMRUREHKYwFhERcZjCWERExGFRhbExZr4x5n1jzBZjzHciTDfGmF8Gp79njDkr9kUVERFJTB2GsTHGDTwIFAETgKuMMRPCZisCxgUfNwAPxbicIiIiCSuaI+NpwBZr7VZrbR2wDFgYNs9C4HEb8AbQzxiTG+OyioiIJKRowngo8EnI68rguM7OIyIiIhF4opjHRBhnT2IejDE3EOjGBqgxxrwfxftHayCwN4brc5LqEp8SpS6JUg9QXeJVotSlO+oxMtLIaMK4Ehge8noYsPMk5sFauxRYGsV7dpoxZq21dmp3rLunqS7xKVHqkij1ANUlXiVKXXqyHtF0U68BxhljRhtjkoDFwLNh8zwLfDF4VvV04JC1tirGZRUREUlIHR4ZW2sbjDFfB1YCbuBRa+0GY8yNwekPA8XAAmALcAxY0n1FFhERSSzRdFNjrS0mELih4x4OGbbATbEtWqd1S/e3Q1SX+JQodUmUeoDqEq8SpS49Vg8TyFERERFxii6HKSIi4rBeEcZduRxnR8v2tCjqcnWwDu8ZY/5ljCkMmbbdGFNmjFlvjFnbsyVvVc6O6jHLGHMoWNb1xpg7o122p0VRl2+H1KPcGOMzxmQFp8XTZ/KoMWa3Maa8jem9aTvpqC69YjsJlqejuvSmbaWjuvSWbWW4MWaVMWaTMWaDMebfI8zTs9uLtTauHwROGvsQGAMkAe8CE8LmWQCUEPi983TgzWiXjcO6nAf0Dw4XNdYl+Ho7MLCXfCazgOdOZtl4q0vY/JcCL8fbZxIsy0zgLKC8jem9YjuJsi5xv510oi69YluJpi5h88bztpILnBUczgAqnM6V3nBk3JXLcUazbE/qsDzW2n9Zaw8EX75B4Dfb8aYrf9de95mEuQr4U4+UrJOstauB/e3M0lu2kw7r0ku2EyCqz6Utve5zCRPP20qVtfbt4PARYBOtrxrZo9tLbwjjrlyOM94u09nZ8nyFQMuskQVeMMasM4GrmTkl2nqca4x51xhTYoyZ2Mlle0rU5THGpAHzgadDRsfLZxKN3rKddFa8bied0Ru2laj1pm3FGDMKmAy8GTapR7eXqH7a5LCuXI4zqst09qCoy2OMmU1gJ3N+yOgZ1tqdxpjBwIvGmM3BlmpPi6YebwMjrbU1xpgFwN8I3NWr134mBLrdXrPWhh4ZxMtnEo3esp1ELc63k2j1lm2lM3rFtmKMSSfQYPimtfZw+OQIi3Tb9tIbjoy7cjnOqC7T2YOiKo8xZhLwCLDQWruvcby1dmfweTewnEB3iRM6rIe19rC1tiY4XAx4jTEDo1m2h3WmPIsJ63aLo88kGr1lO4lKL9hOotKLtpXOiPttxRjjJRDET1hrn4kwS89uL058ed6ZB4Gj963AaJq/LJ8YNs+naflF+1vRLhuHdRlB4Epm54WN7wNkhAz/C5gfx/XIofl37NOAj4OfT6/7TILzZRL4rqxPPH4mIWUaRdsnCvWK7STKusT9dtKJuvSKbSWaugSnx/22Evz7Pg78dzvz9Oj2Evfd1LYLl+Nsa1kHqkF75Qmry53AAOB/jDEADTZwofJsYHlwnAd40lq7woFqRFuPRcBXjTENwHFgsQ38J/fGzwTgs8AL1tqjIYvHzWcCYIz5E4EzcwcaYyqBHwBe6F3bCURVl7jfThpFUZdesa1AVHWBXrCtADOAa4AyY8z64Lg7CDTyHNledAUuERERh/WG74xFREQSmsJYRETEYQpjERERhymMRUREHKYwFhERcZjCWERExGEKYxEREYcpjEVERBz2/wGpQnLGai60awAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "model.save_weights('Bert_stepword_lemmatizer.h5')\n",
    "pd.DataFrame(history.history).plot(figsize=(8, 5))\n",
    "plt.grid(True)\n",
    "#plt.gca().set_xlim(0,33)\n",
    "plt.gca().set_ylim(0,1)\n",
    "loss, accuracy = model.evaluate({'input_ids':x_val['input_ids'],'attention_mask':x_val['attention_mask']}, labels_val\n",
    "    )\n",
    "print('Accuracy: %f' % (accuracy*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ffd42dec",
   "metadata": {},
   "source": [
    "### <font color='violet'>Load the weights</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "6128844c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#rebuild the model to load the weights\n",
    "\n",
    "input_ids = Input(shape=(max_length,), dtype=tf.int32, name=\"input_ids\")\n",
    "input_mask = Input(shape=(max_length,), dtype=tf.int32, name=\"attention_mask\")\n",
    "# embeddings = dbert_model(input_ids,attention_mask = input_mask)[0]\n",
    "\n",
    "\n",
    "embeddings = bert(input_ids,attention_mask = input_mask)[0] #(0 is the last hidden states,1 is the pooler_output)\n",
    "x = tf.keras.layers.GlobalMaxPool1D()(embeddings)\n",
    "x = Dense(138, activation='elu',kernel_initializer='GlorotNormal')(x)\n",
    "x = tf.keras.layers.Dropout(0.1)(x)\n",
    "x = Dense(28,activation = 'elu',kernel_initializer='GlorotNormal')(x)\n",
    "\n",
    "output = Dense(6,activation = 'softmax')(x)\n",
    "    \n",
    "model_saved = tf.keras.Model(inputs=[input_ids, input_mask], outputs=output)\n",
    "model_saved.layers[2].trainable = True\n",
    "\n",
    "\n",
    "model_saved.load_weights('Bert_stepword_lemmatizer.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "wx9sRqqxYKtN",
   "metadata": {
    "id": "wx9sRqqxYKtN"
   },
   "source": [
    "### <font color='violet'>More details about the model</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "79679347",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_ids (InputLayer)          [(None, 43)]         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "attention_mask (InputLayer)     [(None, 43)]         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "tf_bert_model_1 (TFBertModel)   TFBaseModelOutputWit 108310272   input_ids[0][0]                  \n",
      "                                                                 attention_mask[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_1 (GlobalM (None, 768)          0           tf_bert_model_1[1][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 138)          106122      global_max_pooling1d_1[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dropout_75 (Dropout)            (None, 138)          0           dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 28)           3892        dropout_75[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 6)            174         dense_4[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 108,420,460\n",
      "Trainable params: 108,420,460\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "0 <tensorflow.python.keras.engine.input_layer.InputLayer object at 0x000001E8F7425610>\n",
      "   no activation attribute\n",
      "1 <tensorflow.python.keras.engine.input_layer.InputLayer object at 0x000001E9C529F5B0>\n",
      "   no activation attribute\n",
      "2 <transformers.models.bert.modeling_tf_bert.TFBertModel object at 0x000001E9D9B7E100>\n",
      "   no activation attribute\n",
      "3 <tensorflow.python.keras.layers.pooling.GlobalMaxPooling1D object at 0x000001E9C530FAC0>\n",
      "   no activation attribute\n",
      "4 <tensorflow.python.keras.layers.core.Dense object at 0x000001E9C52DCFA0>\n",
      "     <function elu at 0x000001E8FE087D30>\n",
      "5 <tensorflow.python.keras.layers.core.Dropout object at 0x000001E9C93CE9A0>\n",
      "   no activation attribute\n",
      "6 <tensorflow.python.keras.layers.core.Dense object at 0x000001E8898FC310>\n",
      "     <function elu at 0x000001E8FE087D30>\n",
      "7 <tensorflow.python.keras.layers.core.Dense object at 0x000001E9F8BACF40>\n",
      "     <function softmax at 0x000001E8FDA1B700>\n",
      "0   <tensorflow.python.keras.engine.input_layer.InputLayer object at 0x000001E8F7425610>: \n",
      "{'batch_input_shape': (None, 43), 'dtype': 'int32', 'sparse': False, 'ragged': False, 'name': 'input_ids'} \n",
      "\n",
      "1   <tensorflow.python.keras.engine.input_layer.InputLayer object at 0x000001E9C529F5B0>: \n",
      "{'batch_input_shape': (None, 43), 'dtype': 'int32', 'sparse': False, 'ragged': False, 'name': 'attention_mask'} \n",
      "\n",
      "2   <transformers.models.bert.modeling_tf_bert.TFBertModel object at 0x000001E9D9B7E100>: \n",
      "{'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': None, 'use_bfloat16': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'chunk_size_feed_forward': 0, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'architectures': ['BertForMaskedLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': None, 'pad_token_id': 0, 'eos_token_id': None, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'bert-base-cased', 'transformers_version': '4.19.1', 'gradient_checkpointing': False, 'model_type': 'bert', 'vocab_size': 28996, 'hidden_size': 768, 'num_hidden_layers': 12, 'num_attention_heads': 12, 'hidden_act': 'gelu', 'intermediate_size': 3072, 'hidden_dropout_prob': 0.1, 'attention_probs_dropout_prob': 0.1, 'max_position_embeddings': 512, 'type_vocab_size': 2, 'initializer_range': 0.02, 'layer_norm_eps': 1e-12, 'position_embedding_type': 'absolute', 'use_cache': True, 'classifier_dropout': None} \n",
      "\n",
      "3   <tensorflow.python.keras.layers.pooling.GlobalMaxPooling1D object at 0x000001E9C530FAC0>: \n",
      "{'name': 'global_max_pooling1d', 'trainable': True, 'dtype': 'float32', 'data_format': 'channels_last'} \n",
      "\n",
      "4   <tensorflow.python.keras.layers.core.Dense object at 0x000001E9C52DCFA0>: \n",
      "{'name': 'dense', 'trainable': True, 'dtype': 'float32', 'units': 138, 'activation': 'elu', 'use_bias': True, 'kernel_initializer': {'class_name': 'GlorotNormal', 'config': {'seed': None}}, 'bias_initializer': {'class_name': 'Zeros', 'config': {}}, 'kernel_regularizer': None, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None} \n",
      "\n",
      "5   <tensorflow.python.keras.layers.core.Dropout object at 0x000001E9C93CE9A0>: \n",
      "{'name': 'dropout_74', 'trainable': True, 'dtype': 'float32', 'rate': 0.1, 'noise_shape': None, 'seed': None} \n",
      "\n",
      "6   <tensorflow.python.keras.layers.core.Dense object at 0x000001E8898FC310>: \n",
      "{'name': 'dense_1', 'trainable': True, 'dtype': 'float32', 'units': 28, 'activation': 'elu', 'use_bias': True, 'kernel_initializer': {'class_name': 'GlorotNormal', 'config': {'seed': None}}, 'bias_initializer': {'class_name': 'Zeros', 'config': {}}, 'kernel_regularizer': None, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None} \n",
      "\n",
      "7   <tensorflow.python.keras.layers.core.Dense object at 0x000001E9F8BACF40>: \n",
      "{'name': 'dense_2', 'trainable': True, 'dtype': 'float32', 'units': 6, 'activation': 'softmax', 'use_bias': True, 'kernel_initializer': {'class_name': 'GlorotUniform', 'config': {'seed': None}}, 'bias_initializer': {'class_name': 'Zeros', 'config': {}}, 'kernel_regularizer': None, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None} \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'name': 'Adam',\n",
       " 'clipnorm': 1.0,\n",
       " 'learning_rate': 5e-05,\n",
       " 'decay': 0.01,\n",
       " 'beta_1': 0.9,\n",
       " 'beta_2': 0.999,\n",
       " 'epsilon': 1e-08,\n",
       " 'amsgrad': False}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_saved.summary()\n",
    "\n",
    "#to visualize activation functions\n",
    "for i, layer in enumerate (model.layers):\n",
    "    print (i, layer)\n",
    "    try:\n",
    "        print (\"    \",layer.activation)\n",
    "    except AttributeError:\n",
    "        print('   no activation attribute')\n",
    "#specific info about each layer\n",
    "for i in range(len(model.layers)):\n",
    "    print(f'{i}   {model.layers[i]}: \\n{model.layers[i].get_config()} \\n')\n",
    "#info about optimizers\n",
    "model.optimizer.get_config()        "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1e40f524",
   "metadata": {},
   "source": [
    "### <font color='violet'>Tokenize the test dataset</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b91dfd21",
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test = tokenizer(\n",
    "    [x.split() for x in test['sentences']],\n",
    "    add_special_tokens=True,\n",
    "    max_length=max_length,\n",
    "    truncation=True,\n",
    "    padding=True, \n",
    "    return_tensors='tf',\n",
    "    return_token_type_ids = False,\n",
    "    return_attention_mask = True,\n",
    "    is_split_into_words=True,\n",
    "    verbose = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8222bea",
   "metadata": {},
   "source": [
    "### <font color='violet'>Predict on test dataset</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "86e246dd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "63/63 [==============================] - 6s 90ms/step - loss: 0.1699 - accuracy: 0.9295\n",
      "Accuracy: 92.949998\n"
     ]
    }
   ],
   "source": [
    "loss, accuracy = model.evaluate({'input_ids':x_test['input_ids'],'attention_mask':x_test['attention_mask']}, labels_test\n",
    "    )\n",
    "print('Accuracy: %f' % (accuracy*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "69d497b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.config.experimental.list_physical_devices('GPU')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "4742b343",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{4: 'sadness', 0: 'anger', 3: 'love', 5: 'surprise', 1: 'fear', 2: 'joy'}"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#class-label conversion\n",
    "fin_labels=[i.replace(\"\\n\", \"\") for i in train.loc[:,'emotion'].to_list()]\n",
    "dict(zip(labels_train,fin_labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ffcdf69",
   "metadata": {},
   "source": [
    "### <font color='violet'>Predict a new sentence</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "id": "3ccc9d63",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I am glad to see you again\n"
     ]
    }
   ],
   "source": [
    "y=input()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "id": "e8fd792f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['joy'], dtype='<U8')"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_s=pd.Series([y])\n",
    "y_lemm=y_s.apply(lambda x: Lemmatizer_stop_word(x))\n",
    "y_tok = tokenizer(\n",
    "    [x.split() for x in y_lemm],\n",
    "    add_special_tokens=True,\n",
    "    max_length=max_length,\n",
    "    truncation=True,\n",
    "    padding='max_length',  #only for sentence prediction \n",
    "    return_tensors='tf',\n",
    "    return_token_type_ids = False,\n",
    "    return_attention_mask = True,\n",
    "    is_split_into_words=True,\n",
    "    verbose = True)\n",
    "#labels_y=lb.transform(test.loc[:,'emotion'].to_list())\n",
    "y_prob=model.predict({'input_ids':y_tok['input_ids'],'attention_mask':y_tok['attention_mask']})*100\n",
    "#y_tok\n",
    "class_label=y_prob.argmax(axis=-1)\n",
    "lb.inverse_transform(class_label) #from class to label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "id": "9ab7a465",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': <tf.Tensor: shape=(1, 43), dtype=int32, numpy=\n",
       " array([[ 101,  178, 5171, 1267,  102,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "            0,    0,    0,    0,    0,    0,    0,    0,    0,    0]])>,\n",
       " 'attention_mask': <tf.Tensor: shape=(1, 43), dtype=int32, numpy=\n",
       " array([[1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])>}"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "{'input_ids':y_tok['input_ids'],'attention_mask':y_tok['attention_mask']} #bert input parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "072c39da",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Copy of Pretrained_BERT.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "0354ebd23e8b43d7b20507ce61ced75f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_576b4d397e344e6db9d8ceb827b7f30b",
      "max": 29,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_e9551198bd294828beb08c2344a4ca40",
      "value": 29
     }
    },
    "0926f474a4ee4e4a8a25df14e8010057": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_16ffcfecc4694d96b320c984420092db",
      "max": 570,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_0b25fce2aebf4b618d3d25befcbb548d",
      "value": 570
     }
    },
    "0b20c48d5f464045bc00233d756f7394": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_cb1a226cefc04fed9985ad80af06ae0c",
       "IPY_MODEL_0926f474a4ee4e4a8a25df14e8010057",
       "IPY_MODEL_48c90aafded445179ce1790db5034791"
      ],
      "layout": "IPY_MODEL_12977d5c6b9d4959b73e5e275f58156d"
     }
    },
    "0b25fce2aebf4b618d3d25befcbb548d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "0c475dfefd6642fdbea19926a8f2b1a2": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0db5a207c5784e1ba416a210f963f6ed": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "11e879c146f74eaca50b87e533b0ef94": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_add58dda103842bb9a457789e10ba815",
      "placeholder": "​",
      "style": "IPY_MODEL_7dcab70ff4cc47f2a949891fb0307958",
      "value": "Downloading: 100%"
     }
    },
    "124676f8d87d47cb8ed99c5bdbc2c709": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "12977d5c6b9d4959b73e5e275f58156d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1322931022824693b2fa808f16345f4f": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "134be23324084589a61ab8b3597b011c": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "16ffcfecc4694d96b320c984420092db": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "200054adda5a4e448ff8cd42c0c4b979": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3d26457d9f424a04b85fd4bbf3d82952": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "418b186a0a084f3dadd3d2c9a6c7320d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "424eb576b30b413d856ac5d05109a333": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "44ae46594ad6419d8a8718b75cc8631a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_4950944610b247ae86e519786030f59c",
      "placeholder": "​",
      "style": "IPY_MODEL_418b186a0a084f3dadd3d2c9a6c7320d",
      "value": " 426k/426k [00:00&lt;00:00, 856kB/s]"
     }
    },
    "48c90aafded445179ce1790db5034791": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_0c475dfefd6642fdbea19926a8f2b1a2",
      "placeholder": "​",
      "style": "IPY_MODEL_fcf54e89ec3b450289334968e72c1696",
      "value": " 570/570 [00:00&lt;00:00, 12.0kB/s]"
     }
    },
    "4950944610b247ae86e519786030f59c": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "50800c3e605848b1a06f2454ad41d45a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_890d83f54eff47389a97b19a87480558",
      "max": 213450,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_acc0bf6ad46e458292ea8548674ba3d2",
      "value": 213450
     }
    },
    "54908bd5c07d40f5a54b3fd9c71390eb": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_6d0f234011664804942cd59fdc9692fa",
      "placeholder": "​",
      "style": "IPY_MODEL_b76dfd538a2048afb422de054cb649ac",
      "value": " 29.0/29.0 [00:00&lt;00:00, 553B/s]"
     }
    },
    "576b4d397e344e6db9d8ceb827b7f30b": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "57d69d7e33544816b8e0d354942670ae": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "669bea4aef8d491cb2d225a935917a99": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_a18bc3fb32e24a6cae43286b8525965f",
      "placeholder": "​",
      "style": "IPY_MODEL_8e11b818377340bb9b18c0aae1625ceb",
      "value": "Downloading: 100%"
     }
    },
    "690e502115044d598eaa98a58efce5cb": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1322931022824693b2fa808f16345f4f",
      "placeholder": "​",
      "style": "IPY_MODEL_124676f8d87d47cb8ed99c5bdbc2c709",
      "value": "Downloading: 100%"
     }
    },
    "6d0f234011664804942cd59fdc9692fa": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "765aa75236ab4e929313703e683e9a12": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "7dcab70ff4cc47f2a949891fb0307958": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "83f3212a35a74383a58a682b9e118924": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "890d83f54eff47389a97b19a87480558": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8e11b818377340bb9b18c0aae1625ceb": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "8fe3507fab5945fc8612a9aead7da428": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_669bea4aef8d491cb2d225a935917a99",
       "IPY_MODEL_bddcf68e529546999493a4a416ae2a1c",
       "IPY_MODEL_44ae46594ad6419d8a8718b75cc8631a"
      ],
      "layout": "IPY_MODEL_3d26457d9f424a04b85fd4bbf3d82952"
     }
    },
    "97c739b4bc0a4ef0944994fa2a80b66b": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "99b9f086a3594466845ac6158df6ebe2": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a03821876f924ca090316cc8abb0b4b9": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_99b9f086a3594466845ac6158df6ebe2",
      "placeholder": "​",
      "style": "IPY_MODEL_83f3212a35a74383a58a682b9e118924",
      "value": "Downloading: 100%"
     }
    },
    "a18bc3fb32e24a6cae43286b8525965f": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "acc0bf6ad46e458292ea8548674ba3d2": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "add58dda103842bb9a457789e10ba815": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ae5b0d2f26be4b3aad87f273589b8949": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b6d732c4381145819856422dc9eca800": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_11e879c146f74eaca50b87e533b0ef94",
       "IPY_MODEL_0354ebd23e8b43d7b20507ce61ced75f",
       "IPY_MODEL_54908bd5c07d40f5a54b3fd9c71390eb"
      ],
      "layout": "IPY_MODEL_97c739b4bc0a4ef0944994fa2a80b66b"
     }
    },
    "b76dfd538a2048afb422de054cb649ac": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "bddcf68e529546999493a4a416ae2a1c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ae5b0d2f26be4b3aad87f273589b8949",
      "max": 435797,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_c843f6f684df4df1a364be495d12b7c5",
      "value": 435797
     }
    },
    "c843f6f684df4df1a364be495d12b7c5": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "cb1a226cefc04fed9985ad80af06ae0c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ef7282e944e14b5ea845c9c2bba91163",
      "placeholder": "​",
      "style": "IPY_MODEL_f20625027b1f469aa778e82d9bf021fa",
      "value": "Downloading: 100%"
     }
    },
    "ccc309f621234b48babe6e5f39c636bf": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_a03821876f924ca090316cc8abb0b4b9",
       "IPY_MODEL_50800c3e605848b1a06f2454ad41d45a",
       "IPY_MODEL_ef894c59eb414f738714a021496a4dd8"
      ],
      "layout": "IPY_MODEL_d391c6c6285644229f6d74cdc6d790df"
     }
    },
    "d30bc051413d401facbbb93b32827130": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_dcf068b3b8fe473e87c42b28bc328f7d",
      "max": 526681800,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_0db5a207c5784e1ba416a210f963f6ed",
      "value": 526681800
     }
    },
    "d391c6c6285644229f6d74cdc6d790df": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "dcf068b3b8fe473e87c42b28bc328f7d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e9551198bd294828beb08c2344a4ca40": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "ec4ea935b1b3418db19e3af6b422916b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_690e502115044d598eaa98a58efce5cb",
       "IPY_MODEL_d30bc051413d401facbbb93b32827130",
       "IPY_MODEL_f31f896390d245aaac88c491bfe011c2"
      ],
      "layout": "IPY_MODEL_200054adda5a4e448ff8cd42c0c4b979"
     }
    },
    "ef7282e944e14b5ea845c9c2bba91163": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ef894c59eb414f738714a021496a4dd8": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_424eb576b30b413d856ac5d05109a333",
      "placeholder": "​",
      "style": "IPY_MODEL_57d69d7e33544816b8e0d354942670ae",
      "value": " 208k/208k [00:00&lt;00:00, 923kB/s]"
     }
    },
    "f20625027b1f469aa778e82d9bf021fa": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "f31f896390d245aaac88c491bfe011c2": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_134be23324084589a61ab8b3597b011c",
      "placeholder": "​",
      "style": "IPY_MODEL_765aa75236ab4e929313703e683e9a12",
      "value": " 502M/502M [00:11&lt;00:00, 43.4MB/s]"
     }
    },
    "fcf54e89ec3b450289334968e72c1696": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}