DepressionDetectionInTwitter.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Depression Detection in Social Media Posts\n",
    "\n",
    "#### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import ftfy\n",
    "import matplotlib.pyplot as plt\n",
    "import nltk\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re\n",
    "\n",
    "from math import exp\n",
    "from numpy import sign\n",
    "\n",
    "from sklearn.metrics import  classification_report, confusion_matrix, accuracy_score\n",
    "from gensim.models import KeyedVectors\n",
    "from nltk.corpus import stopwords\n",
    "from nltk import PorterStemmer\n",
    "\n",
    "from keras.models import Model, Sequential\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "from keras.layers import Conv1D, Dense, Input, LSTM, Embedding, Dropout, Activation, MaxPooling1D\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reproducibility\n",
    "np.random.seed(1234)\n",
    "\n",
    "DEPRES_NROWS = 3200  # number of rows to read from DEPRESSIVE_TWEETS_CSV\n",
    "RANDOM_NROWS = 12000 # number of rows to read from RANDOM_TWEETS_CSV\n",
    "MAX_SEQUENCE_LENGTH = 140 # Max tweet size\n",
    "MAX_NB_WORDS = 20000\n",
    "EMBEDDING_DIM = 300\n",
    "TRAIN_SPLIT = 0.6\n",
    "TEST_SPLIT = 0.2\n",
    "LEARNING_RATE = 0.1\n",
    "EPOCHS= 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Section 1: Load Data\n",
    "\n",
    "Loading depressive tweets scraped from twitter using [TWINT](https://github.com/haccer/twint) and random tweets from Kaggle dataset [twitter_sentiment](https://www.kaggle.com/ywang311/twitter-sentiment/data).\n",
    "\n",
    "#### File Paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#DEPRESSIVE_TWEETS_CSV = 'depressive_tweets.csv'\n",
    "DEPRESSIVE_TWEETS_CSV = 'depressive_tweets_processed.csv'\n",
    "RANDOM_TWEETS_CSV = 'Sentiment Analysis Dataset 2.csv'\n",
    "EMBEDDING_FILE = 'GoogleNews-vectors-negative300.bin.gz'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "depressive_tweets_df = pd.read_csv(DEPRESSIVE_TWEETS_CSV, sep = '|', header = None, usecols = range(0,9), nrows = DEPRES_NROWS)\n",
    "random_tweets_df = pd.read_csv(RANDOM_TWEETS_CSV, encoding = \"ISO-8859-1\", usecols = range(0,4), nrows = RANDOM_NROWS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>989292962323615744</td>\n",
       "      <td>2018-04-25</td>\n",
       "      <td>23:59:57</td>\n",
       "      <td>Eastern Standard Time</td>\n",
       "      <td>whosalli</td>\n",
       "      <td>The lack of this understanding is a small but ...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>989292959844663296</td>\n",
       "      <td>2018-04-25</td>\n",
       "      <td>23:59:56</td>\n",
       "      <td>Eastern Standard Time</td>\n",
       "      <td>estermnunes</td>\n",
       "      <td>i just told my parents about my depression and...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>989292951716155392</td>\n",
       "      <td>2018-04-25</td>\n",
       "      <td>23:59:54</td>\n",
       "      <td>Eastern Standard Time</td>\n",
       "      <td>TheAlphaAries</td>\n",
       "      <td>depression is something i don't speak about ev...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>989292873664393218</td>\n",
       "      <td>2018-04-25</td>\n",
       "      <td>23:59:35</td>\n",
       "      <td>Eastern Standard Time</td>\n",
       "      <td>_ojhodgson</td>\n",
       "      <td>Made myself a tortilla filled with pb&amp;j. My de...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>989292856119472128</td>\n",
       "      <td>2018-04-25</td>\n",
       "      <td>23:59:31</td>\n",
       "      <td>Eastern Standard Time</td>\n",
       "      <td>DMiller96371630</td>\n",
       "      <td>@WorldofOutlaws I am gonna need depression med...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    0           1         2                      3  \\\n",
       "0  989292962323615744  2018-04-25  23:59:57  Eastern Standard Time   \n",
       "1  989292959844663296  2018-04-25  23:59:56  Eastern Standard Time   \n",
       "2  989292951716155392  2018-04-25  23:59:54  Eastern Standard Time   \n",
       "3  989292873664393218  2018-04-25  23:59:35  Eastern Standard Time   \n",
       "4  989292856119472128  2018-04-25  23:59:31  Eastern Standard Time   \n",
       "\n",
       "                 4                                                  5  6  7  8  \n",
       "0         whosalli  The lack of this understanding is a small but ...  1  0  3  \n",
       "1      estermnunes  i just told my parents about my depression and...  1  0  2  \n",
       "2    TheAlphaAries  depression is something i don't speak about ev...  0  0  0  \n",
       "3       _ojhodgson  Made myself a tortilla filled with pb&j. My de...  1  0  0  \n",
       "4  DMiller96371630  @WorldofOutlaws I am gonna need depression med...  0  0  0  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "depressive_tweets_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ItemID</th>\n",
       "      <th>Sentiment</th>\n",
       "      <th>SentimentSource</th>\n",
       "      <th>SentimentText</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Sentiment140</td>\n",
       "      <td>is so sad for my APL frie...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>Sentiment140</td>\n",
       "      <td>I missed the New Moon trail...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>Sentiment140</td>\n",
       "      <td>omg its already 7:30 :O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>Sentiment140</td>\n",
       "      <td>.. Omgaga. Im sooo  im gunna CRy. I'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>Sentiment140</td>\n",
       "      <td>i think mi bf is cheating on me!!!   ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ItemID  Sentiment SentimentSource  \\\n",
       "0       1          0    Sentiment140   \n",
       "1       2          0    Sentiment140   \n",
       "2       3          1    Sentiment140   \n",
       "3       4          0    Sentiment140   \n",
       "4       5          0    Sentiment140   \n",
       "\n",
       "                                       SentimentText  \n",
       "0                       is so sad for my APL frie...  \n",
       "1                     I missed the New Moon trail...  \n",
       "2                            omg its already 7:30 :O  \n",
       "3            .. Omgaga. Im sooo  im gunna CRy. I'...  \n",
       "4           i think mi bf is cheating on me!!!   ...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random_tweets_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Section 2: Data Processing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Pretrained Word2Vec Model\n",
    "\n",
    "The pretrained vectors for the Word2Vec model is from [here](https://drive.google.com/file/d/0B7XkCwpI5KDYNlNUTTlSS21pQmM/edit).\n",
    "Using a Keyed Vectors file, we can get the embedding of any word by calling `.word_vec(word)` and we can get all the words in the model's vocabulary through `.vocab`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2vec = KeyedVectors.load_word2vec_format(EMBEDDING_FILE, binary=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing\n",
    "\n",
    "Preprocessing the tweets in order to:\n",
    "* Remove links and images\n",
    "* Remove hashtags\n",
    "* Remove @ mentions\n",
    "* Remove emojis\n",
    "* Remove stop words\n",
    "* Remove punctuation\n",
    "* Get rid of stuff like \"what's\" and making it \"what is'\n",
    "* Stem words so they are all the same tense (e.g. ran -> run)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Expand Contraction\n",
    "cList = {\n",
    "  \"ain't\": \"am not\",\n",
    "  \"aren't\": \"are not\",\n",
    "  \"can't\": \"cannot\",\n",
    "  \"can't've\": \"cannot have\",\n",
    "  \"'cause\": \"because\",\n",
    "  \"could've\": \"could have\",\n",
    "  \"couldn't\": \"could not\",\n",
    "  \"couldn't've\": \"could not have\",\n",
    "  \"didn't\": \"did not\",\n",
    "  \"doesn't\": \"does not\",\n",
    "  \"don't\": \"do not\",\n",
    "  \"hadn't\": \"had not\",\n",
    "  \"hadn't've\": \"had not have\",\n",
    "  \"hasn't\": \"has not\",\n",
    "  \"haven't\": \"have not\",\n",
    "  \"he'd\": \"he would\",\n",
    "  \"he'd've\": \"he would have\",\n",
    "  \"he'll\": \"he will\",\n",
    "  \"he'll've\": \"he will have\",\n",
    "  \"he's\": \"he is\",\n",
    "  \"how'd\": \"how did\",\n",
    "  \"how'd'y\": \"how do you\",\n",
    "  \"how'll\": \"how will\",\n",
    "  \"how's\": \"how is\",\n",
    "  \"I'd\": \"I would\",\n",
    "  \"I'd've\": \"I would have\",\n",
    "  \"I'll\": \"I will\",\n",
    "  \"I'll've\": \"I will have\",\n",
    "  \"I'm\": \"I am\",\n",
    "  \"I've\": \"I have\",\n",
    "  \"isn't\": \"is not\",\n",
    "  \"it'd\": \"it had\",\n",
    "  \"it'd've\": \"it would have\",\n",
    "  \"it'll\": \"it will\",\n",
    "  \"it'll've\": \"it will have\",\n",
    "  \"it's\": \"it is\",\n",
    "  \"let's\": \"let us\",\n",
    "  \"ma'am\": \"madam\",\n",
    "  \"mayn't\": \"may not\",\n",
    "  \"might've\": \"might have\",\n",
    "  \"mightn't\": \"might not\",\n",
    "  \"mightn't've\": \"might not have\",\n",
    "  \"must've\": \"must have\",\n",
    "  \"mustn't\": \"must not\",\n",
    "  \"mustn't've\": \"must not have\",\n",
    "  \"needn't\": \"need not\",\n",
    "  \"needn't've\": \"need not have\",\n",
    "  \"o'clock\": \"of the clock\",\n",
    "  \"oughtn't\": \"ought not\",\n",
    "  \"oughtn't've\": \"ought not have\",\n",
    "  \"shan't\": \"shall not\",\n",
    "  \"sha'n't\": \"shall not\",\n",
    "  \"shan't've\": \"shall not have\",\n",
    "  \"she'd\": \"she would\",\n",
    "  \"she'd've\": \"she would have\",\n",
    "  \"she'll\": \"she will\",\n",
    "  \"she'll've\": \"she will have\",\n",
    "  \"she's\": \"she is\",\n",
    "  \"should've\": \"should have\",\n",
    "  \"shouldn't\": \"should not\",\n",
    "  \"shouldn't've\": \"should not have\",\n",
    "  \"so've\": \"so have\",\n",
    "  \"so's\": \"so is\",\n",
    "  \"that'd\": \"that would\",\n",
    "  \"that'd've\": \"that would have\",\n",
    "  \"that's\": \"that is\",\n",
    "  \"there'd\": \"there had\",\n",
    "  \"there'd've\": \"there would have\",\n",
    "  \"there's\": \"there is\",\n",
    "  \"they'd\": \"they would\",\n",
    "  \"they'd've\": \"they would have\",\n",
    "  \"they'll\": \"they will\",\n",
    "  \"they'll've\": \"they will have\",\n",
    "  \"they're\": \"they are\",\n",
    "  \"they've\": \"they have\",\n",
    "  \"to've\": \"to have\",\n",
    "  \"wasn't\": \"was not\",\n",
    "  \"we'd\": \"we had\",\n",
    "  \"we'd've\": \"we would have\",\n",
    "  \"we'll\": \"we will\",\n",
    "  \"we'll've\": \"we will have\",\n",
    "  \"we're\": \"we are\",\n",
    "  \"we've\": \"we have\",\n",
    "  \"weren't\": \"were not\",\n",
    "  \"what'll\": \"what will\",\n",
    "  \"what'll've\": \"what will have\",\n",
    "  \"what're\": \"what are\",\n",
    "  \"what's\": \"what is\",\n",
    "  \"what've\": \"what have\",\n",
    "  \"when's\": \"when is\",\n",
    "  \"when've\": \"when have\",\n",
    "  \"where'd\": \"where did\",\n",
    "  \"where's\": \"where is\",\n",
    "  \"where've\": \"where have\",\n",
    "  \"who'll\": \"who will\",\n",
    "  \"who'll've\": \"who will have\",\n",
    "  \"who's\": \"who is\",\n",
    "  \"who've\": \"who have\",\n",
    "  \"why's\": \"why is\",\n",
    "  \"why've\": \"why have\",\n",
    "  \"will've\": \"will have\",\n",
    "  \"won't\": \"will not\",\n",
    "  \"won't've\": \"will not have\",\n",
    "  \"would've\": \"would have\",\n",
    "  \"wouldn't\": \"would not\",\n",
    "  \"wouldn't've\": \"would not have\",\n",
    "  \"y'all\": \"you all\",\n",
    "  \"y'alls\": \"you alls\",\n",
    "  \"y'all'd\": \"you all would\",\n",
    "  \"y'all'd've\": \"you all would have\",\n",
    "  \"y'all're\": \"you all are\",\n",
    "  \"y'all've\": \"you all have\",\n",
    "  \"you'd\": \"you had\",\n",
    "  \"you'd've\": \"you would have\",\n",
    "  \"you'll\": \"you you will\",\n",
    "  \"you'll've\": \"you you will have\",\n",
    "  \"you're\": \"you are\",\n",
    "  \"you've\": \"you have\"\n",
    "}\n",
    "\n",
    "c_re = re.compile('(%s)' % '|'.join(cList.keys()))\n",
    "\n",
    "def expandContractions(text, c_re=c_re):\n",
    "    def replace(match):\n",
    "        return cList[match.group(0)]\n",
    "    return c_re.sub(replace, text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_tweets(tweets):\n",
    "    cleaned_tweets = []\n",
    "    for tweet in tweets:\n",
    "        tweet = str(tweet)\n",
    "        # if url links then dont append to avoid news articles\n",
    "        # also check tweet length, save those > 10 (length of word \"depression\")\n",
    "        if re.match(\"(\\w+:\\/\\/\\S+)\", tweet) == None and len(tweet) > 10:\n",
    "            #remove hashtag, @mention, emoji and image URLs\n",
    "            tweet = ' '.join(re.sub(\"(@[A-Za-z0-9]+)|(\\#[A-Za-z0-9]+)|(<Emoji:.*>)|(pic\\.twitter\\.com\\/.*)\", \" \", tweet).split())\n",
    "            \n",
    "            #fix weirdly encoded texts\n",
    "            tweet = ftfy.fix_text(tweet)\n",
    "            \n",
    "            #expand contraction\n",
    "            tweet = expandContractions(tweet)\n",
    "\n",
    "            #remove punctuation\n",
    "            tweet = ' '.join(re.sub(\"([^0-9A-Za-z \\t])\", \" \", tweet).split())\n",
    "\n",
    "            #stop words\n",
    "            stop_words = set(stopwords.words('english'))\n",
    "            word_tokens = nltk.word_tokenize(tweet) \n",
    "            filtered_sentence = [w for w in word_tokens if not w in stop_words]\n",
    "            tweet = ' '.join(filtered_sentence)\n",
    "\n",
    "            #stemming words\n",
    "            tweet = PorterStemmer().stem(tweet)\n",
    "            \n",
    "            cleaned_tweets.append(tweet)\n",
    "\n",
    "    return cleaned_tweets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Applying the preprocessing `clean_text` function to every element in the depressive tweets and random tweets data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "depressive_tweets_arr = [x for x in depressive_tweets_df[5]]\n",
    "random_tweets_arr = [x for x in random_tweets_df['SentimentText']]\n",
    "X_d = clean_tweets(depressive_tweets_arr)\n",
    "X_r = clean_tweets(random_tweets_arr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenizer\n",
    "\n",
    "Using a Tokenizer to assign indices and filtering out unfrequent words. Tokenizer creates a map of every unique word and an assigned index to it. The parameter called num_words indicates that we only care about the top 20000 most frequent words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer(num_words=MAX_NB_WORDS)\n",
    "tokenizer.fit_on_texts(X_d + X_r)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Applying the tokenizer to depressive tweets and random tweets data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "sequences_d = tokenizer.texts_to_sequences(X_d)\n",
    "sequences_r = tokenizer.texts_to_sequences(X_r)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Number of unique words in tokenizer. Has to be <= 20,000."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 21548 unique tokens\n"
     ]
    }
   ],
   "source": [
    "word_index = tokenizer.word_index\n",
    "print('Found %s unique tokens' % len(word_index))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pad sequences all to the same length of 140 words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of data_d tensor: (2308, 140)\n",
      "Shape of data_r tensor: (11911, 140)\n"
     ]
    }
   ],
   "source": [
    "data_d = pad_sequences(sequences_d, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "data_r = pad_sequences(sequences_r, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "print('Shape of data_d tensor:', data_d.shape)\n",
    "print('Shape of data_r tensor:', data_r.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Embedding Matrix\n",
    "\n",
    "The embedding matrix is a `n x m` matrix where `n` is the number of words and `m` is the dimension of the embedding. In this case, `m=300` and `n=20000`. We take the min between the number of unique words in our tokenizer and max words in case there are less unique words than the max we specified."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "nb_words = min(MAX_NB_WORDS, len(word_index))\n",
    "\n",
    "embedding_matrix = np.zeros((nb_words, EMBEDDING_DIM))\n",
    "\n",
    "for (word, idx) in word_index.items():\n",
    "    if word in word2vec.vocab and idx < MAX_NB_WORDS:\n",
    "        embedding_matrix[idx] = word2vec.word_vec(word)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Splitting and Formatting Data\n",
    "\n",
    "Assigning labels to the depressive tweets and random tweets data, and splitting the arrays into test (60%), validation (20%), and train data (20%). Combine depressive tweets and random tweets arrays and shuffle."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assigning labels to the depressive tweets and random tweets data\n",
    "labels_d = np.array([1] * DEPRES_NROWS)\n",
    "labels_r = np.array([0] * RANDOM_NROWS)\n",
    "\n",
    "# Splitting the arrays into test (60%), validation (20%), and train data (20%)\n",
    "perm_d = np.random.permutation(len(data_d))\n",
    "idx_train_d = perm_d[:int(len(data_d)*(TRAIN_SPLIT))]\n",
    "idx_test_d = perm_d[int(len(data_d)*(TRAIN_SPLIT)):int(len(data_d)*(TRAIN_SPLIT+TEST_SPLIT))]\n",
    "idx_val_d = perm_d[int(len(data_d)*(TRAIN_SPLIT+TEST_SPLIT)):]\n",
    "\n",
    "perm_r = np.random.permutation(len(data_r))\n",
    "idx_train_r = perm_r[:int(len(data_r)*(TRAIN_SPLIT))]\n",
    "idx_test_r = perm_r[int(len(data_r)*(TRAIN_SPLIT)):int(len(data_r)*(TRAIN_SPLIT+TEST_SPLIT))]\n",
    "idx_val_r = perm_r[int(len(data_r)*(TRAIN_SPLIT+TEST_SPLIT)):]\n",
    "\n",
    "# Combine depressive tweets and random tweets arrays\n",
    "data_train = np.concatenate((data_d[idx_train_d], data_r[idx_train_r]))\n",
    "labels_train = np.concatenate((labels_d[idx_train_d], labels_r[idx_train_r]))\n",
    "data_test = np.concatenate((data_d[idx_test_d], data_r[idx_test_r]))\n",
    "labels_test = np.concatenate((labels_d[idx_test_d], labels_r[idx_test_r]))\n",
    "data_val = np.concatenate((data_d[idx_val_d], data_r[idx_val_r]))\n",
    "labels_val = np.concatenate((labels_d[idx_val_d], labels_r[idx_val_r]))\n",
    "\n",
    "# Shuffling\n",
    "perm_train = np.random.permutation(len(data_train))\n",
    "data_train = data_train[perm_train]\n",
    "labels_train = labels_train[perm_train]\n",
    "perm_test = np.random.permutation(len(data_test))\n",
    "data_test = data_test[perm_test]\n",
    "labels_test = labels_test[perm_test]\n",
    "perm_val = np.random.permutation(len(data_val))\n",
    "data_val = data_val[perm_val]\n",
    "labels_val = labels_val[perm_val]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Section 3: Building the Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Building Model (LSTM + CNN)\n",
    "\n",
    "The model takes in an input and then outputs a single number representing the probability that the tweet indicates depression. The model takes in each input sentence, replace it with it's embeddings, then run the new embedding vector through a convolutional layer. CNNs are excellent at learning spatial structure from data, the convolutional layer takes advantage of that and learn some structure from the sequential data then pass into a standard LSTM layer. Last but not least, the output of the LSTM layer is fed into a standard Dense model for prediction."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\peijo\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\util\\deprecation.py:497: calling conv1d (from tensorflow.python.ops.nn_ops) with data_format=NHWC is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "`NHWC` for data_format is deprecated, use `NWC` instead\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "# Embedded layer\n",
    "model.add(Embedding(len(embedding_matrix), EMBEDDING_DIM, weights=[embedding_matrix], \n",
    "                            input_length=MAX_SEQUENCE_LENGTH, trainable=False))\n",
    "# Convolutional Layer\n",
    "model.add(Conv1D(filters=32, kernel_size=3, padding='same', activation='relu'))\n",
    "model.add(MaxPooling1D(pool_size=2))\n",
    "model.add(Dropout(0.2))\n",
    "# LSTM Layer\n",
    "model.add(LSTM(300))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(1, activation='sigmoid'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compiling Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 140, 300)          6000000   \n",
      "_________________________________________________________________\n",
      "conv1d_1 (Conv1D)            (None, 140, 32)           28832     \n",
      "_________________________________________________________________\n",
      "max_pooling1d_1 (MaxPooling1 (None, 70, 32)            0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 70, 32)            0         \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 300)               399600    \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 300)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 301       \n",
      "=================================================================\n",
      "Total params: 6,428,733\n",
      "Trainable params: 428,733\n",
      "Non-trainable params: 6,000,000\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model.compile(loss='binary_crossentropy', optimizer='nadam', metrics=['acc'])\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Section 4: Training the Model\n",
    "\n",
    "The model is trained `EPOCHS` time, and Early Stopping argument is used to end training if the loss or accuracy don't improve within 3 epochs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8530 samples, validate on 2845 samples\n",
      "Epoch 1/10\n",
      "8530/8530 [==============================] - 32s 4ms/step - loss: 0.1128 - acc: 0.9673 - val_loss: 0.0357 - val_acc: 0.9930\n",
      "Epoch 2/10\n",
      "8530/8530 [==============================] - 32s 4ms/step - loss: 0.0378 - acc: 0.9916 - val_loss: 0.0326 - val_acc: 0.9926\n",
      "Epoch 3/10\n",
      "8530/8530 [==============================] - 33s 4ms/step - loss: 0.0308 - acc: 0.9926 - val_loss: 0.0333 - val_acc: 0.9933\n",
      "Epoch 4/10\n",
      "8530/8530 [==============================] - 32s 4ms/step - loss: 0.0250 - acc: 0.9945 - val_loss: 0.0432 - val_acc: 0.9902\n",
      "Epoch 5/10\n",
      "8530/8530 [==============================] - 32s 4ms/step - loss: 0.0214 - acc: 0.9950 - val_loss: 0.0395 - val_acc: 0.9919\n"
     ]
    }
   ],
   "source": [
    "early_stop = EarlyStopping(monitor='val_loss', patience=3)\n",
    "\n",
    "hist = model.fit(data_train, labels_train, \\\n",
    "        validation_data=(data_val, labels_val), \\\n",
    "        epochs=EPOCHS, batch_size=40, shuffle=True, \\\n",
    "        callbacks=[early_stop])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Summarize history for accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEWCAYAAABMoxE0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8VfWd//HXJ/tOIAlrgICggIqgERWse6da96VWq62orb+x0+kyv3aqs3Qcf/XXzvycTsfpamvcN0q12g7uRS0gCioqsmsChDXsIWTP5/fHOUluQiAXkpub5f18PO4j557zPed+7oF7P/f7/Z7z/Zq7IyIicrQS4h2AiIj0bUokIiLSJUokIiLSJUokIiLSJUokIiLSJUokIiLSJUokIodhZg+Z2Q+jLFtmZhfEOiaR3kaJREREukSJRGQAMLOkeMcg/ZcSifR5YZPS98zsQzOrMrMHzGyYmb1gZpVm9qqZDY4of5mZfWxme8zsdTObHLFtupm9F+73NJDW7rUuMbNl4b6LzGxqlDFebGbvm9k+M9toZne1235meLw94fbZ4fp0M/sPM1tvZnvNbEG47hwzK+/gPFwQLt9lZnPN7DEz2wfMNrMZZvZW+BpbzOxnZpYSsf/xZvaKme0ys21m9g9mNtzMDphZXkS5U8yswsySo3nv0v8pkUh/cTXwWeBY4FLgBeAfgHyC/+ffBDCzY4EngW8DBcA84I9mlhJ+qf4BeBQYAvwuPC7hvicDJcD/AvKAXwPPm1lqFPFVAV8BcoGLgdvN7IrwuGPCeP87jGkasCzc717gFGBmGNPfA01RnpPLgbnhaz4ONALfCc/JGcD5wNfDGLKBV4EXgZHABOA1d98KvA5cG3HcG4Gn3L0+yjikn1Mikf7iv919m7tvAv4CvO3u77t7LfAsMD0s90Xgf9z9lfCL8F4gneCL+nQgGfipu9e7+1xgScRrfA34tbu/7e6N7v4wUBvud1ju/rq7f+TuTe7+IUEyOzvcfAPwqrs/Gb7uTndfZmYJwC3At9x9U/iai8L3FI233P0P4WtWu/u77r7Y3RvcvYwgETbHcAmw1d3/w91r3L3S3d8Otz1MkDwws0TgeoJkKwIokUj/sS1iubqD51nh8khgffMGd28CNgKjwm2bvO1IpusjlscC/ztsGtpjZnuA0eF+h2Vmp5nZ/LBJaC/w1wQ1A8JjfNLBbvkETWsdbYvGxnYxHGtmfzKzrWFz1/+NIgaA54ApZjaeoNa3193fOcqYpB9SIpGBZjNBQgDAzIzgS3QTsAUYFa5rNiZieSNwj7vnRjwy3P3JKF73CeB5YLS7DwJ+BTS/zkbgmA722QHUHGJbFZAR8T4SCZrFIrUf2vuXwCpgorvnEDT9dRYD7l4DzCGoOX0Z1UakHSUSGWjmABeb2flhZ/H/JmieWgS8BTQA3zSzJDO7CpgRse9vgL8OaxdmZplhJ3p2FK+bDexy9xozmwF8KWLb48AFZnZt+Lp5ZjYtrC2VAD8xs5FmlmhmZ4R9MmuAtPD1k4F/Ajrrq8kG9gH7zWwScHvEtj8Bw83s22aWambZZnZaxPZHgNnAZcBjUbxfGUCUSGRAcffVBO39/03wi/9S4FJ3r3P3OuAqgi/M3QT9Kc9E7LuUoJ/kZ+H2dWHZaHwduNvMKoEfECS05uNuAD5PkNR2EXS0nxRu/i7wEUFfzS7g34AEd98bHvO3BLWpKqDNVVwd+C5BAqskSIpPR8RQSdBsdSmwFVgLnBuxfSFBJ/97Yf+KSAvTxFYiEg0z+zPwhLv/Nt6xSO+iRCIinTKzU4FXCPp4KuMdj/QuatoSkcMys4cJ7jH5tpKIdEQ1EhER6RLVSEREpEsGxEBu+fn5XlRUFO8wRET6lHfffXeHu7e/P+kgAyKRFBUVsXTp0niHISLSp5jZ+s5LqWlLRES6SIlERES6RIlERES6ZED0kXSkvr6e8vJyampq4h1Kv5CWlkZhYSHJyZrrSGSgGbCJpLy8nOzsbIqKimg72KscKXdn586dlJeXM27cuHiHIyI9LKZNW2Z2oZmtNrN1ZnZHB9vHmtlrFkyR+rqZFUZs+zczWx4+vhix/iEzKw2nO11mZtOOJraamhry8vKURLqBmZGXl6fancgAFbNEEs6P8HPgImAKcL2ZTWlX7F7gEXefCtwN/Cjc92LgZIIpR08DvmdmORH7fc/dp4WPZRwlJZHuo3MpMnDFsmlrBrDO3T8FMLOnCOaQXhFRZgrBHNIA8wnmy25e/4a7NwANZvYBcCERQ2+LiAw07k5tQxPVdY0cqG/kQG0DB+oaOVDXSHV9A1W1jcG2ugaq6oLlW88cx+DMlJjGFctEMoq2U32WE9QuIn0AXA38F3AlkG1meeH6fzGznxDMAncubRPQPWb2A+A14I6O5rA2s9uA2wDGjBnTfnPc7dmzhyeeeIKvf/3rR7Tf5z//eZ544glyc3NjFJmIdFV9Y1Pw5R5+qTd/2UcuV4fLVRHL1XWNVLVZPnhb0xEMj2gGl08b2acTSUdtHe1PwXeBn5nZbOBNggl6Gtz95XDY6kVABa0z1wHcSTDxTgpwP/B9gmaxti/kfn+4neLi4l43MuWePXv4xS9+cVAiaWxsJDEx8ZD7zZs3L9ahiQwITU0e/Kqvawi+pGuDX/Xtv/Srw+dVYbmDE0LbbdV1jdQ1Nh1RLOnJiWSkJJKeEvzNSEkiIyWR3Ixk0lOSyOxgW1A+clvr+ubl1KSEHml2jmUiKSeYC7tZIcF82S3cfTPBjHSYWRZwdTjzG+5+D3BPuO0JghnbcPct4e61ZvYgQTLqc+644w4++eQTpk2bRnJyMllZWYwYMYJly5axYsUKrrjiCjZu3EhNTQ3f+ta3uO2224DW4V7279/PRRddxJlnnsmiRYsYNWoUzz33HOnp6XF+ZyKxUV3XSPnuA+yvjfhCj2jeqQ6TQkvzTmTTT33wyz5IFkG5mvoj+7JPSUwgIzWRjOTgizszNYn05ETys1IYk5LR5su9/Rd6ekoimSlJLckgcjk9OZGEhL7dxxjLRLIEmGhm4whqGtfRdp5qzCyfYB7rJoKaRkm4PhHIdfedZjYVmAq8HG4b4e5bLEizVwDLuxrov/7xY1Zs3tfVw7QxZWQO/3Lp8Yfc/uMf/5jly5ezbNkyXn/9dS6++GKWL1/ecvlsSUkJQ4YMobq6mlNPPZWrr76avLy8NsdYu3YtTz75JL/5zW+49tpr+f3vf8+NN97Yre9DpKc1Njnrd1axemslq7ZWsnprJau3VVK2s4rOZr1ITLA2X+LpyYlkpiYyKD2ZETlpQSJoty09JYmMsEaQkZrU8uWe0ZwsUoLkkZSo+7cPJWaJxN0bzOwbwEtAIlDi7h+b2d3AUnd/HjgH+JGZOUHT1t+EuycDfwmrZPuAG8OOd4DHzayAoOlsGfDXsXoPPWnGjBlt7sG47777ePbZZwHYuHEja9euPSiRjBs3jmnTgqufTznlFMrKynosXpHuUFFZGyaMfS0JY822ypbaghkU5WVy3LBsLp82knH5meSkJUc087T91Z+S2DNNOdJWTG9IdPd5wLx2634QsTwXmNvBfjUEV251dMzzujnMw9YcekpmZmbL8uuvv86rr77KW2+9RUZGBuecc06H92ikpqa2LCcmJlJdXd0jsYocqQN1DazZtp/VW/e11jK2VrKzqq6lTH5WKpOGZ3PDaWM5bng2k4ZnM3FoNukph+4zlN5hwN7ZHm/Z2dlUVnY8a+nevXsZPHgwGRkZrFq1isWLF/dwdCJHp7HJKWvTLBXUNNbvOtDSLJWenMixw7K4YPKwloRx3PBs8rJSD39w6bWUSOIkLy+PWbNmccIJJ5Cens6wYcNatl144YX86le/YurUqRx33HGcfvrpcYxU5GDuTsX+sFlqS5g0tu1j7bb91DYEzVIJBkX5mUwZmcOV0wtbksaYIRl9vnNZ2hoQc7YXFxd7+4mtVq5cyeTJk+MUUf+kc9o/VdU2sGZb5UGd37simqUKsoNmqaB2kcOk4dlMGJpFWrKapfoyM3vX3Ys7K6caiYgA0NDYRNnOAy0d381JY8OuAy1lMlISOXZYNn81JWiWCmoZOQyJ8Q1v0rspkYgMMO7O9sralj6M5oSxdvt+6iKapcblZ3LiqEF84ZTCloRRODhdzVJyECUSke7iDvXVULMHqvdAzd6I5UP8TUyGCefDsRdCwaTgetdutD9sllq1JSJpbKtkz4H6ljLDclI5bngOsybkc9ywoJahZik5EkokIpHcoa7q8AmgZu+htzXWHf74qYMgbRCkD4K03GC/V+8KHrljg4Ry7Oeg6ExIiv4qpobGJkp3VLXULpo7vzfuar0kPDMlkWOHZ3PRCSNarpQ6blh2zMdhkv5PieRw6qqgqTH4lWgJ4a/FhNbllnXW7b8kpQvcoXZfuy/5KGoHzUmiqeEwB7cwEeQGiSA9F3JGHryuo79pgyChg1/5ezfB2pdhzUvw3iPwzq8hOROOOTdILBP/CrKHhW/N2bavllURTVKrtlbyyfb9LeM7JSYY4/MzOakwly8Wj27p/B6Vq2YpiQ0lksOp3Bp8IUXDEggSSgeJxhIIElC02yPKHeqYLeX76RdDUxPUHuaX/+FqBzV7wQ8zjpIlHvzFnzv2EAlgUNt1qTmQ0M1DZQwaBcU3B4/6aij9C6x5kaY1L5Kw6k8AlGdMZmFCMc/sP4G3awppHhN1eE4axw3P5qxj84NaxrAcjhmaSWqSmqWk5yiRHM6gUdA0PPhScg//hss0tVsf/qWJYJzniO1NjeAN4fb2x4nu8uusibPYv3Yhm7dW8M1//nfm/ub/hVtaE805V93MvXf9PcXTT+TgxBU8/+kvS7jt5hvJyMgEMz5/zY08UfJrcgfnHj6htUleCdElsMaGdjWB3VHWDvaGCfww5yYhue0XfkY+DDmm81pBei6kZPWqBFzfpllqH6u2DGbV1kvYtOc8JtlGzkt4j7+qWsYX7DG+iHNgUAH7x5xP5gkXkzn5dEjJ7PxFRGJIieRwktJi/xodJaL267wp+PLOHcPIQYXMffqJdvuEfxMSg87bhOSIJFZPZPL66S9+w42Xnk3GkGA+k3kP/j9gH+w50kEr2ycbg8pt8MuvtSaFuo7v3G+RmNr2Cz57BAyd3HmtID0XkjN6VTI4lMYmZ2dVLdv31bK9sib8W8u2fTVsr6xl464DfFpR1dIslZRgjC/I5OSxg/nSaWOYNPxUjht+E6Ny07GqHbDuFTLWvEjGuv+BtU/B86kw7qygX+XYz0Fu75t7R/o/JZI4+f73v8/YsWOD+Ugskbvuugsz480332T37t3U19fzwx/+kMsvv7x1p4w8ysrKuOSSS1i+fDnV1dXcfPPNrFixgsmTJ1Nd7zCoEPKO4fbbb2fJkiVUV1dzzTXX8K//+q/cd999bN5WwbnXf5P8/Hzmv/YaRePHs/TtReTn5fGT//wpJQ89DMBXZ3+Fb//t7ZSVlXHRFV/gzDNOY9HiJYwaMZznniohPS2tbc2qOZENKoThJ3ReK0gbBMl9d8j7hsYmdlbVBQmhXXLY3vy3soYd++to7GAmoiGZKQzNTmVkbjrnHDe0pfP7mIIsUpIO0XSWVQDTvhQ8Gupgw6KgX2X1CzDvFZj3XRh6fJhULoTC4o77ZES6me5sB3jhDtj6Ufe+6PAT4aIfH3Lz+++/z7e//W3eeOMNAKZMmcKLL75Ibm4uOTk57Nixg9NPP521a9diZmRlZbF///42ieQnP/kJy5cvp6SkhA8//JCTTz6ZxYsXU1xczK5duxgyZAiNjY2cf/753HfffUydOrVlPpP8/HygdX6T9evXM3v2bBYvXoy7c9ppp/HYY48xePBgJkyYwNKlS5k2bRrXXnstl112WYfD1feHO9vrG5uoqKxtSQjbKmup2FfDtuYaRWUt2/bVsrOq9qAhzc0gLzOFodlpDM1JZWh2KsNy0hianUpBdhrDclIZmpNGQVbqoZPF0XCHnetgzYtBYlm/CLwRMvKCjvpjPwfHnBckb5EjoDvbe7np06ezfft2Nm/eTEVFBYMHD2bEiBF85zvf4c033yQhIYFNmzaxbds2hg8f3uEx3nzzTb75zW8CMHXqVKZOndqybc6cOdx///00NDSwZcsWVqxY0WZ7ewsWLODKK69sGYX4qquu4i9/+QuXXXZZvxiuvrahsU2CaKlBRNQmKipr24xG2yzBIC8rlWE5QWKYWjiIguy0NoliaE4q+VmpJMdjzgozyJ8YPGb+bdCs+MlrQVJZ8yJ88CQkJMHYmeHlxRdC3jE9H6f0W0okcNiaQyxdc801zJ07l61bt3Ldddfx+OOPU1FRwbvvvktycjJFRUUdDh8fqaO5F0pLS7n33ntZsmQJgwcPZvbs2Z0e53A10948XH1NfWNr/0Ob5qXIPokadkfcgNcsMcEoyAqSQOHgDE4eO7htcghrEUMyU/rWpEbpuXDC1cGjqRHKl7TWVl76h+CRN6H1npUxZwR9ayJHSYkkjq677jq+9rWvsWPHDt544w3mzJnD0KFDSU5OZv78+axfv/6w+5911lk8/vjjnHvuuSxfvpwPP/wQgH379pGZmcmgQYPYtm0bL7zwAueccw7QOnx9c9NW5LFmz57NHXfcgbvz7LPP8uijj8bkfUfjQF3DwX0PEYlh+75g/b6ag+/5SE4MEkRBThpj8zI4ddxghrU0N7X+HZKZQmJ/v68iIRHGnB48LrgLdq8P71l5Ed65H976WXBJc/Pd9RM+C5l5nR1VeqOmJtizHipWQ8Wq8O9K+NLvgv61GFIiiaPjjz+eyspKRo0axYgRI7jhhhu49NJLKS4uZtq0aUyaNOmw+99+++3cfPPNTJ06lWnTpjFjxgwATjrpJKZPn87xxx/P+PHjmTVrVss+t912GxdddBEjRoxg/vz5LetPPvlkZs+e3XKMr371q0yfPr3bm7H21zYEfQ9hQqhoV4vYVllDxb5aKmsPThApiQkUZAdNTMcUZDHzmLygz6FNLSKVwRkpuvHuUAaPhRlfCx61+6H0jdbaysfPAgajZ7R22A+d0ieujhtQmhphd1lromhJHGugIaK1IHsEFBwXXEof40SiznbpNitWrGBPytB2tYjW/odt+2o4UNd40H6pSQlt+hqaaw2RtYhhOakMSk/WNKqx0tQEWz+A1S8GiWXLsmD9oNGtSaXoM5DcA5fES6CxAXaXhkkirGFsXwU71kBjbWu5nFHBOG0Fk4LE0fw3PbfLIaizXXrc7gP13PDI2y3P05MTgyuVstM4fmQO5x43NLxyqTU5FGSnkZOWpAQRbwkJMHJ68Dj3Tti3pXXYlmVPwJLfBvfujD+nddiWnBHxjrp/aKyHXZ8GyWJ7RNLYubbt2G2DxgQJYvzZQbIYOhnyj4W0nPjFHlIikW5R19BIdV0j188Yw1c/M46h2alkpSpB9Fk5I+CUm4JHfQ2ULWhtAls9LygzYlprh/2Iad0/dEx/01AXXKbd0n8RJo2d69qO75Y7NkgUEy9orV3kHwup2fGLvRMDumlr0qRJ+qLrJpt3H2D5ylWcePxkRgzquzcaSifcYfvK1qRS/k5wM2rWsPCelQuDWktqVrwjjZ/6moiEEVnD+CS4vwcAgyHj2jVHTQou4e5FQ96oaasTaWlp7Ny5k7y8PCWTLmpobGLr9gpqmhKURPo7Mxg2JXh85u+gaiesezVILCueh/cfhcSUoD+lubYyeGy8o46N+mrYsbZdH8bKoF+jedBQS4Ah44MkMfmyiBrGxD49skN7A7ZGUl9fT3l5eaf3V0jn9tc0sGxLFcXHT2D62PzOd5D+qbEeNixura3sXBusL5gcMWzLqZDYx36/1lUFHdzNzVHN/Ri7y2gZWDQhKRg0tLl2MTSsYeRNOKJ5ZXqbaGskAzaRSPdoanLO/Y/XyctM4Zmvz+p8Bxk4dn7Senf9+oVBP0D64OBelWM/F9y7kj443lG2qt0PO1a37fCuWAV7NtCaMJKD5DC03VVSQ46BpP43QZiatqRHvLZqO+t3HuB7nzsu3qFIb5N3DJzx9eBRsxc+mR8klrUvwUdzgnlhxs5sra3kTeiZe1Zq9gb3XLTvw9i7sbVMYkrQwV1YDNNvbO3DGDJOowB0QDUS6ZLr71/M+p1VvPn35/atYUQkfpoaYdO7rU1g25YH64eMjxi2ZWbXf+FX7253l3fYLFW5ubVMUlrQX9FyH0b4GFzU95rgYkA1Eom5FZv38danO7nzoklKIhK9hMTg7vnRM+D8H8CejUEtZc1LsOQBWPwLSMmGCee1DttyuDuzD+xqrVlENkvt39paJjkjqGGMOytojho6OfibO1ZD7XeDmCYSM7sQ+C8gEfitu/+43faxQAlQAOwCbnT38nDbvwEXh0X/j7s/Ha4fBzwFDAHeA77s7gcP2SoxV7KwlIyURK47VZMpSRfkjoZTvxo86qqg9M3W2sqK5wALmpiO/Vxww+Su0rb3YVRVtB4rOTNIEBPOb3uX96Axus8lhmKWSMwsEfg58FmgHFhiZs+7+4qIYvcCj7j7w2Z2HvAj4MtmdjFwMjANSAXeMLMX3H0f8G/Af7r7U2b2K+BW4Jexeh/SsYrKWp5ftpnrZoxmUIbajKWbpGTCcRcFD3fY+mFrh/2ff9haLjUnSBDHfi5MFmENI2eUEkYcxLJGMgNY5+6fApjZU8DlQGQimQJ8J1yeD/whYv0b7t4ANJjZB8CFZvY74DzgS2G5h4G7UCLpcY+/vZ66xiZmzyyKdyjSX5nBiJOCx9l/H0zlvGN1cIVUzkgNJtmLxDJ1jwIiLoOgPFwX6QPg6nD5SiDbzPLC9ReZWYaZ5QPnAqOBPGBPmGAOdUwAzOw2M1tqZksrKio6KiJHqaa+kccWr+f8SUMZXzCA72CWnpU9LOjjGDRKSaSXiWUi6ehfuv0lYt8Fzjaz94GzgU1Ag7u/DMwDFgFPAm8BDVEeM1jpfr+7F7t7cUFBbIdQHmj++MFmduyv45Yzx8U7FBHpBWKZSMoJahHNCoHNkQXcfbO7X+Xu04F/DNftDf/e4+7T3P2zBAlkLbADyDWzpEMdU2LL3SlZWMak4dnMPEYTIIlIbBPJEmCimY0zsxTgOuD5yAJmlm9mzTHcSXAFF2aWGDZxYWZTganAyx7c9DIfuCbc5ybguRi+B2nnrU93snLLPm6ZNU5jlIkIEMNEEvZjfAN4CVgJzHH3j83sbjO7LCx2DrDazNYAw4B7wvXJwF/MbAVwP8Flwc39It8H/s7M1hH0mTwQq/cgBytZUMaQzBQumzYy3qGISC8R0/tI3H0eQV9H5LofRCzPBeZ2sF8NwZVbHR3zU4IrwqSHle2o4rVV2/jbcyeQlqybuEQkoAuuJWoPLSojKcG48Yx+Oiy4iBwVJRKJyt7qeuYs3cilJ41kaLbm7RaRVkokEpU5SzZyoK6RW2bpkl8RaUuJRDrV0NjEQ4vKOG3cEE4YNSje4YhIL6NEIp16ZcU2Nu2p1g2IItIhJRLp1AMLShkzJIMLJg+Ldygi0gspkchhfbBxD0vX72b2zCISE3QDoogcTIlEDuvBhaVkpSbxheLCeIciIr2UEokc0ta9Nfzpwy1cWzya7DTNOSIiHVMikUN6dHEZTe7cPKso3qGISC+mRCIdqq5r5Im3N/DZKcMYPSQj3uGISC+mRCIdevb9Tew+UK8bEEWkU0okcpBgzpFSThiVw4xxQ+Idjoj0ckokcpC/rN3Buu37NeeIiERFiUQO8sCCUgqyU7lkquYcEZHOKZFIG+u2V/LGmgq+cvpYUpL030NEOqdvCmmjZGEZKUkJfOm0MfEORUT6CCUSabG7qo5n3ivnqumjyMtKjXc4ItJHKJFIiyeXbKCmvombdcmviBwBJRIBoL6xiUcWrefMCfkcNzw73uGISB+iRCIAzPtoC1v31XCr5hwRkSOkRCLBDYgLShmfn8nZxxbEOxwR6WOUSIT3Nuzmg/K93DyriATNOSIiR0iJRChZUEZOWhJXn6I5R0TkyCmRDHDluw/wwvItXH/aGDJSkuIdjoj0QUokA9wjb63HzLjpjKJ4hyIifVRME4mZXWhmq81snZnd0cH2sWb2mpl9aGavm1lhxLZ/N7OPzWylmd1n4eiBYbnVZrYsfAyN5Xvoz6pqG3jynQ1cdMJwRuamxzscEemjYpZIzCwR+DlwETAFuN7MprQrdi/wiLtPBe4GfhTuOxOYBUwFTgBOBc6O2O8Gd58WPrbH6j30d79/r5zKmgZu0SW/ItIFsayRzADWufun7l4HPAVc3q7MFOC1cHl+xHYH0oAUIBVIBrbFMNYBp6nJeXBhGdNG53LymMHxDkdE+rBYJpJRwMaI5+XhukgfAFeHy1cC2WaW5+5vESSWLeHjJXdfGbHfg2Gz1j/bISbMMLPbzGypmS2tqKjojvfTr8xfvZ3SHVW6AVFEuiyWiaSjL3hv9/y7wNlm9j5B09UmoMHMJgCTgUKC5HOemZ0V7nODu58IfCZ8fLmjF3f3+9292N2LCwp0k117JQtLGTEojQtPGB7vUESkj4tlIikHRkc8LwQ2RxZw983ufpW7Twf+MVy3l6B2stjd97v7fuAF4PRw+6bwbyXwBEETmhyBlVv2sXDdTr5yRhHJibpwT0S6JpbfIkuAiWY2zsxSgOuA5yMLmFm+mTXHcCdQEi5vIKipJJlZMkFtZWX4PD/cNxm4BFgew/fQLz24sJT05ESunzG688IiIp2IWSJx9wbgG8BLwEpgjrt/bGZ3m9llYbFzgNVmtgYYBtwTrp8LfAJ8RNCP8oG7/5Gg4/0lM/sQWEbQFPabWL2H/mjH/lr+sGwzV58yityMlHiHIyL9QExvZXb3ecC8dut+ELE8lyBptN+vEfhfHayvAk7p/kgHjscXb6CuQXOOiEj3UQP5AFLb0Miji9dz7nEFHFOQFe9wRKSfUCIZQP74wRZ27K/VDYgi0q2iSiRm9nszuziiY1z6mOY5R44dlsWZE/LjHY6I9CPRJoZfAl8C1prZj81sUgxjkhh4u3QXK7bs45ZZ4zjEPZwiIkclqkTi7q+6+w3AyUAZ8IqMlDb9AAAXHUlEQVSZLTKzm8PLcKWXe2BBKYMzkrlievvBBUREuibqpiozywNmA18F3gf+iyCxvBKTyKTbrN9Zxasrt3HDaWNJS06Mdzgi0s9EdfmvmT0DTAIeBS519y3hpqfNbGmsgpPu8dCiMpISjC+fMTbeoYhIPxTtfSQ/c/c/d7TB3Yu7MR7pZvtq6pmzZCOXTB3JsJy0eIcjIv1QtE1bk80st/mJmQ02s6/HKCbpRnOWbKSqrpFbdAOiiMRItInka+6+p/mJu+8GvhabkKS7NDY5Dy0qY0bREE4sHBTvcESkn4o2kSREzvsRzn6ogZp6uVdWbKV8dzW3nFkU71BEpB+Lto/kJWCOmf2KYE6RvwZejFlU0i1KFpRRODidz07RnCMiEjvRJpLvEwyieDvBhFUvA7+NVVDSdR+V7+Wdsl3808WTSUzQDYgiEjtRJRJ3byK4u/2XsQ1HukvJwlIyUxK59lTNOSIisRXtfSQTgR8BU4CWa0jdfXyM4pIu2Lavhj99uJkbTx9LTpoGHhCR2Iq2s/1BgtpIA3Au8AjBzYnSCz361noampzZM4viHYqIDADRJpJ0d38NMHdf7+53AefFLiw5WjX1jTz+9noumDyMsXmZ8Q5HRAaAaDvba8Ih5Nea2TcIprgdGruw5Gj94f1N7D5Qz62ac0REeki0NZJvAxnANwmmur0RuClWQcnRcXdKFpYyZUQOp40bEu9wRGSA6LRGEt58eK27fw/YD9wc86jkqCxYt4M12/Zz7xdO0pwjItJjOq2RuHsjcIrpm6nXK1lQSn5WKpeeNCLeoYjIABJtH8n7wHNm9jugqnmluz8Tk6jkiK3bvp/5qyv4zgXHkpqkOUdEpOdEm0iGADtpe6WWA0okvcRDi0pJSUrghtPHxDsUERlgor2zXf0ivdieA3X8/t1NXDFtJPlZqfEOR0QGmGjvbH+QoAbShrvf0u0RyRF78p2NVNc3crPmHBGROIi2aetPEctpwJXA5u4PR45UfWMTj7xVxsxj8pg8Iife4YjIABRt09bvI5+b2ZPAqzGJSI7Ii8u3smVvDT+84oR4hyIiA1S0NyS2NxHotFfXzC40s9Vmts7M7uhg+1gze83MPjSz182sMGLbv5vZx2a20szua7782MxOMbOPwmPeN9AvS35gQSlFeRmce5wGGhCR+IgqkZhZpZnta34AfySYo+Rw+yQCPwcuIhg1+Hozm9Ku2L3AI+4+FbibYIRhzGwmMAuYCpwAnAqcHe7zS+A2gmQ2EbgwmvfQH723YTfLNu7h5lnjSNCcIyISJ9E2bWUfxbFnAOvc/VMAM3sKuBxYEVFmCvCdcHk+8IfmlyToi0khmEgrGdhmZiOAHHd/KzzmI8AVwAtHEV+f98CCUrLTkrjmlMLOC4uIxEi0NZIrzWxQxPNcM7uik91GARsjnpeH6yJ9AFwdLl8JZJtZXpgo5gNbwsdL7r4y3L+8k2M2x3ibmS01s6UVFRWdhNr3bNpTzYvLt3L9jDFkpkZ7zYSISPeLto/kX9x9b/MTd98D/Esn+3TU1tL+EuLvAmeb2fsETVebgAYzmwBMBgoJEsV5ZnZWlMdsjvF+dy929+KCgoJOQu17HnmrDHfnK2eMjXcoIjLARftTtqOE09m+5UDkPK+FtLtk2N03A1cBmFkWcLW77zWz24DF7r4/3PYCcDrBZFqFhzvmQFBV28CTb2/gohNGUDg4I97hiMgAF22NZKmZ/cTMjjGz8Wb2n8C7neyzBJhoZuPMLAW4Dng+soCZ5YfznADcCZSEyxsIaipJZpZMUFtZ6e5bgEozOz28WusrwHNRvod+45n3ytlX08AtZxbFOxQRkagTyd8CdcDTwBygGvibw+3g7g3AN4CXgJXAHHf/2MzuNrPLwmLnAKvNbA0wDLgnXD8X+AT4iKAf5QN3/2O47Xbgt8C6sMyA6mhvanIeXFjGSaNzOXnM4HiHIyKCuXfYxdCvFBcX+9KlS+MdRrf486pt3PLQUv7rumlcPq3D6wxERLqFmb3r7sWdlYv2qq1XzCw34vlgM3upKwHK0SlZUMbwnDQ+f6LmHBGR3iHapq388EotANx9N5qzvcet3lrJgnU7+MrMsSQnHu2gBCIi3Svab6MmM2sZEsXMijjEZbcSOyULSklLTuD6UzXniIj0HtFe/vuPwAIzeyN8fhbBMCXSQ3bur+XZZZu45pRCBmemxDscEZEW0Q6R8qKZFRMkj2UEl9xWxzIwaeuJtzdQ19DELbOK4h2KiEgb0U5s9VXgWwQ3AC4juDnwLdpOvSsxUtvQyCOL13P2sQVMGHo0w56JiMROtH0k3yIYgXe9u58LTAf63wBWvdT/fLiFispabjlTMyCKSO8TbSKpcfcaADNLdfdVwHGxC0uauTsPLChlwtAszpqYH+9wREQOEm1ne3l4H8kfgFfMbDcDcIyreHindBcfb97H/73yRAb4HF4i0ktF29l+Zbh4l5nNBwYBL8YsKmlRsrCU3Ixkrpyuu9hFpHc64oks3P2NzktJd9iw8wAvr9jG1885hvSUxHiHIyLSId0e3Ys9tKiMRDO+fHpRvEMRETkkJZJeqrKmnjlLN3Lx1BEMH5QW73BERA5JiaSXmrO0nP21DdyqS35FpJdTIumFGpuchxaVUjx2MFMLczvfQUQkjpRIeqFXV25j465q3YAoIn2CEkkv9MCCUkblpvNXU4bFOxQRkU4pkfQyyzft5Z3SXcyeWUSS5hwRkT5A31S9TMnCUjJSErn21NHxDkVEJCpKJL3I9n01/PGDzVxbPJpB6cnxDkdEJCpKJL3IY4vX09DkzJ5ZFO9QRESipkTSS9TUN/LY2xs4f9IwivIz4x2OiEjUlEh6ieeWbWJXVR23nFkU71BERI6IEkkv4O6ULChj0vBszhifF+9wRESOiBJJL7Dok52s3lbJrWeO05wjItLnKJH0Ag8sKCU/K4VLTxoZ71BERI6YEkmcfVqxnz+v2s4Np40lLVlzjohI3xPTRGJmF5rZajNbZ2Z3dLB9rJm9ZmYfmtnrZlYYrj/XzJZFPGrM7Ipw20NmVhqxbVos30OsPbiwjJTEBG48fWy8QxEROSpHPENitMwsEfg58FmgHFhiZs+7+4qIYvcCj7j7w2Z2HvAj4MvuPh+YFh5nCLAOeDliv++5+9xYxd5T9h6oZ+675Vw2bSQF2anxDkdE5KjEskYyA1jn7p+6ex3wFHB5uzJTgNfC5fkdbAe4BnjB3Q/ELNI4eWrJBqrrG7lllkb5FZG+K5aJZBSwMeJ5ebgu0gfA1eHylUC2mbW//vU64Ml26+4Jm8P+08w6/ClvZreZ2VIzW1pRUXF07yCGGhqbeHhRGWeMz2PKyJx4hyMictRimUg6uo7V2z3/LnC2mb0PnA1sAhpaDmA2AjgReClinzuBScCpwBDg+x29uLvf7+7F7l5cUFBw1G8iVl78eCub99ZozhER6fNi1kdCUAOJHMK2ENgcWcDdNwNXAZhZFnC1u++NKHIt8Ky710fssyVcrDWzBwmSUZ9TsqCUsXkZnDdpaLxDERHpkljWSJYAE81snJmlEDRRPR9ZwMzyzaw5hjuBknbHuJ52zVphLQUL7ty7Algeg9hj6v0Nu3lvwx5unllEYoJuQBSRvi1micTdG4BvEDRLrQTmuPvHZna3mV0WFjsHWG1ma4BhwD3N+5tZEUGN5o12h37czD4CPgLygR/G6j3ESsnCMrJTk7imWHOOiEjfF8umLdx9HjCv3bofRCzPBTq8jNfdyzi4cx53P697o+xZW/ZWM++jLdw8s4is1JiefhGRHqE723vYw4vW4+7cpDlHRKSfUCLpQQfqGnjynQ187vjhjB6SEe9wRES6hRJJD3rmvU3sra7XJb8i0q8okfSQpianZGEpUwsHUTx2cLzDERHpNkokPeSNtRV8WlHFLbM054iI9C9KJD2kZEEpQ7NT+fyJI+IdiohIt1Ii6QFrtlXyl7U7uGlmESlJOuUi0r/oW60HPLiwlNSkBK6fMSbeoYiIdDslkhjbVVXHM+9t4qqTRzEkMyXe4YiIdDslkhh74u311DY0ac4REem3lEhiqK6hiUfeWs9nJuYzcVh2vMMREYkJJZIY+p+PNrO9spZbdQOiiPRjSiQx4u48sKCUYwoyOWti75tYS0SkuyiRxMjS9btZvmkfN88aR4LmHBGRfkyJJEYe+Espg9KTufrkwniHIiISU0okMbBx1wFeXrGVL502hvSUxHiHIyISU0okMfDwojISzPjKGWPjHYqISMwpkXSz/bUNPL1kI58/cQQjBqXHOxwRkZhTIulmv1u6kcraBs05IiIDhhJJN2psch5aVMbJY3KZNjo33uGIiPQIJZJu9NrKbazfeYBbzxwf71BERHqMEkk3KllYyshBaXzu+GHxDkVEpMcokXSTjzfvZfGnu7hpZhFJiTqtIjJw6Buvm5QsKCMjJZHrTtWcIyIysCiRdIPtlTX88YPNXHNKIYMykuMdjohIj1Ii6QaPLd5AXWMTs2cWxTsUEZEeF9NEYmYXmtlqM1tnZnd0sH2smb1mZh+a2etmVhiuP9fMlkU8aszsinDbODN728zWmtnTZhbXaQdr6ht5fPF6zp80lPEFWfEMRUQkLmKWSMwsEfg5cBEwBbjezKa0K3Yv8Ii7TwXuBn4E4O7z3X2au08DzgMOAC+H+/wb8J/uPhHYDdwaq/cQjec/2MzOqjrdgCgiA1YsayQzgHXu/qm71wFPAZe3KzMFeC1cnt/BdoBrgBfc/YCZGUFimRtuexi4otsjj5K7U7KglEnDs5l5TF68whARiatYJpJRwMaI5+XhukgfAFeHy1cC2WbW/hv5OuDJcDkP2OPuDYc5Zo9565OdrNpayS2zxhHkOBGRgSeWiaSjb1Zv9/y7wNlm9j5wNrAJaE4SmNkI4ETgpSM4ZvO+t5nZUjNbWlFRcaSxR6VkYSlDMlO4bNrImBxfRKQviGUiKQdGRzwvBDZHFnD3ze5+lbtPB/4xXLc3osi1wLPuXh8+3wHkmlnSoY4Zcez73b3Y3YsLCrp/qtvSHVW8tmo7N542hrRkzTkiIgNXLBPJEmBieJVVCkET1fORBcws38yaY7gTKGl3jOtpbdbC3Z2gL+WacNVNwHMxiL1TDy0sJSnBuFFzjojIABezRBL2Y3yDoFlqJTDH3T82s7vN7LKw2DnAajNbAwwD7mne38yKCGo0b7Q79PeBvzOzdQR9Jg/E6j0cyt7qen73bjmXnjSSodlpPf3yIiK9SlLnRY6eu88D5rVb94OI5bm0XoHVft8yOuhId/dPCa4Ii5unl2zgQF0jt8zSJb8iIrqz/Qg1NDbx8KL1nDZuCCeMGhTvcERE4k6J5Ai9vGIbm/ZU6wZEEZGQEskRemBBKWOGZHDBZM05IiICSiRHZNnGPby7fjezZxaRmKAbEEVEQInkiJQsKCUrNYkvFBfGOxQRkV5DiSRKW/ZWM++jLXzx1NFkp2nOERGRZkokUXr0rfU0uWvOERGRdpRIolBd18gT72zgs1OGMXpIRrzDERHpVZRIovDM++XsOVDPrWeOj3coIiK9jhJJJ5qagjlHThiVw6lFg+MdjohIr6NE0ok311bwSUWV5hwRETkEJZJOlCwsoyA7lUumas4REZGOKJEcxtptlby5poKvnD6WlCSdKhGRjujb8TBKFpaRkpTAl04bE+9QRER6LSWSwxibl8GtZ44jLys13qGIiPRaMZ2PpK/767OPiXcIIiK9nmokIiLSJUokIiLSJUokIiLSJUokIiLSJUokIiLSJUokIiLSJUokIiLSJUokIiLSJebu8Y4h5sysAlh/lLvnAzu6MZzuoriOjOI6MorryPTXuMa6e0FnhQZEIukKM1vq7sXxjqM9xXVkFNeRUVxHZqDHpaYtERHpEiUSERHpEiWSzt0f7wAOQXEdGcV1ZBTXkRnQcamPREREukQ1EhER6RIlEhER6RIlkpCZXWhmq81snZnd0cH2VDN7Otz+tpkV9ZK4ZptZhZktCx9f7YGYSsxsu5ktP8R2M7P7wpg/NLOTYx1TlHGdY2Z7I87VD3oortFmNt/MVprZx2b2rQ7K9Pg5izKuHj9nZpZmZu+Y2QdhXP/aQZke/zxGGVePfx4jXjvRzN43sz91sC2258vdB/wDSAQ+AcYDKcAHwJR2Zb4O/Cpcvg54upfENRv4WQ+fr7OAk4Hlh9j+eeAFwIDTgbd7SVznAH+Kw/+vEcDJ4XI2sKaDf8ceP2dRxtXj5yw8B1nhcjLwNnB6uzLx+DxGE1ePfx4jXvvvgCc6+veK9flSjSQwA1jn7p+6ex3wFHB5uzKXAw+Hy3OB883MekFcPc7d3wR2HabI5cAjHlgM5JrZiF4QV1y4+xZ3fy9crgRWAqPaFevxcxZlXD0uPAf7w6fJ4aP9VUE9/nmMMq64MLNC4GLgt4coEtPzpUQSGAVsjHhezsEfqJYy7t4A7AXyekFcAFeHzSFzzWx0jGOKRrRxx8MZYdPEC2Z2fE+/eNikMJ3g12ykuJ6zw8QFcThnYTPNMmA78Iq7H/J89eDnMZq4ID6fx58Cfw80HWJ7TM+XEkmgo8zc/pdGNGW6WzSv+UegyN2nAq/S+qsjnuJxrqLxHsHYQScB/w38oSdf3MyygN8D33b3fe03d7BLj5yzTuKKyzlz90Z3nwYUAjPM7IR2ReJyvqKIq8c/j2Z2CbDd3d89XLEO1nXb+VIiCZQDkb8cCoHNhypjZknAIGLfjNJpXO6+091rw6e/AU6JcUzRiOZ89jh339fcNOHu84BkM8vvidc2s2SCL+vH3f2ZDorE5Zx1Flc8z1n4mnuA14EL222Kx+ex07ji9HmcBVxmZmUEzd/nmdlj7crE9HwpkQSWABPNbJyZpRB0Rj3frszzwE3h8jXAnz3suYpnXO3a0S8jaOeOt+eBr4RXIp0O7HX3LfEOysyGN7cLm9kMgv//O3vgdQ14AFjp7j85RLEeP2fRxBWPc2ZmBWaWGy6nAxcAq9oV6/HPYzRxxePz6O53unuhuxcRfEf82d1vbFcspucrqbsO1Je5e4OZfQN4ieBKqRJ3/9jM7gaWuvvzBB+4R81sHUEmv66XxPVNM7sMaAjjmh3ruMzsSYKrefLNrBz4F4KOR9z9V8A8gquQ1gEHgJtjHVOUcV0D3G5mDUA1cF0P/BiA4Bfjl4GPwvZ1gH8AxkTEFo9zFk1c8ThnI4CHzSyRIHHNcfc/xfvzGGVcPf55PJSePF8aIkVERLpETVsiItIlSiQiItIlSiQiItIlSiQiItIlSiQiItIlSiQivZwFI/AeNKKrSG+hRCIiIl2iRCLSTczsxnC+imVm9utwgL/9ZvYfZvaemb1mZgVh2Wlmtjgc3O9ZMxscrp9gZq+GgyS+Z2bHhIfPCgcBXGVmj/fAyNMiUVMiEekGZjYZ+CIwKxzUrxG4AcgE3nP3k4E3CO62B3gE+H44uN9HEesfB34eDpI4E2geJmU68G1gCsH8NLNi/qZEoqQhUkS6x/kEA/QtCSsL6QRDjTcBT4dlHgOeMbNBQK67vxGufxj4nZllA6Pc/VkAd68BCI/3jruXh8+XAUXAgti/LZHOKZGIdA8DHnb3O9usNPvnduUONybR4ZqraiOWG9FnV3oRNW2JdI/XgGvMbCiAmQ0xs7EEn7FrwjJfAha4+15gt5l9Jlz/ZeCNcC6QcjO7IjxGqpll9Oi7EDkK+lUj0g3cfYWZ/RPwspklAPXA3wBVwPFm9i7BrHRfDHe5CfhVmCg+pXW03y8Dvw5Hbq0HvtCDb0PkqGj0X5EYMrP97p4V7zhEYklNWyIi0iWqkYiISJeoRiIiIl2iRCIiIl2iRCIiIl2iRCIiIl2iRCIiIl3y/wF/YrAbFB6AhAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1be4fbce080>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(hist.history['acc'])\n",
    "plt.plot(hist.history['val_acc'])\n",
    "plt.title('model accuracy')\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'validation'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Summarize history for loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl4XNWd7vvvr0rzaFuDB8m2ZNmAJyYbx44bbEIgQBJIGgKEkEA6CeB783TnpDPAOZ2kk3tv3/Q9uZk6CWCGNISEIaSTdho4EAI2DsHgASfYTJZt2ZZky7Jsa561zh97SyrLkqtkq7RL0vt5Hj0q7VpV9VPZVa/WWrvWMuccIiIipxIKugAREUl8CgsREYlKYSEiIlEpLEREJCqFhYiIRKWwEBGRqBQWImfIzP7dzP7vGNtWmNkHz/R+REabwkJERKJSWIiISFQKC5kQ/OGfr5rZX82s2cweNLOpZvasmTWa2QtmNjmi/TVmttPMjpvZejObH3HdBWa2zb/dE0DagMf6iJlt92/7ZzM79zRr/oKZlZvZUTNbZ2Yz/ONmZj8ws8NmVu//Tov86642s7f82qrM7Cun9YSJDKCwkInkOuBy4Czgo8CzwH8H8vFeC38PYGZnAY8BXwIKgGeA35tZipmlAL8DfgFMAX7t3y/+bS8EHgLuAPKA+4B1ZpY6nELN7APA/wvcAEwH9gGP+1dfAVzi/x6TgBuBOv+6B4E7nHPZwCLgxeE8rshQFBYykfybc67GOVcFbARec8694ZxrB34LXOC3uxF42jn3B+dcJ/A9IB14P7AcSAZ+6JzrdM49BWyOeIwvAPc5515zznU75x4G2v3bDcengIecc9v8+u4GVphZCdAJZAPnAOace9s5d9C/XSewwMxynHPHnHPbhvm4IoNSWMhEUhNxuXWQn7P8yzPw/pIHwDnXAxwAivzrqtyJK3Dui7g8G/hHfwjquJkdB2b6txuOgTU04fUeipxzLwI/AX4K1JjZWjPL8ZteB1wN7DOzDWa2YpiPKzIohYXIyarx3vQBb44A7w2/CjgIFPnHes2KuHwA+H+cc5MivjKcc4+dYQ2ZeMNaVQDOuR8755YAC/GGo77qH9/snLsWKMQbLntymI8rMiiFhcjJngQ+bGaXmVky8I94Q0l/Bl4FuoC/N7MkM/tbYFnEbe8H7jSz9/kT0Zlm9mEzyx5mDb8CPmtm5/vzHf+CN2xWYWYX+fefDDQDbUC3P6fyKTPL9YfPGoDuM3geRPooLEQGcM69C9wC/BtwBG8y/KPOuQ7nXAfwt8BtwDG8+Y3/iLjtFrx5i5/415f7bYdbwx+BbwC/wevNlAE3+Vfn4IXSMbyhqjq8eRWATwMVZtYA3On/HiJnzLT5kYiIRKOehYiIRKWwEBGRqBQWIiISlcJCRESiSgq6gJGSn5/vSkpKgi5DRGRM2bp16xHnXEG0duMmLEpKStiyZUvQZYiIjClmti96Kw1DiYhIDBQWIiISlcJCRESiGjdzFoPp7OyksrKStra2oEuJu7S0NIqLi0lOTg66FBEZh8Z1WFRWVpKdnU1JSQknLhI6vjjnqKuro7KyktLS0qDLEZFxaFwPQ7W1tZGXlzeugwLAzMjLy5sQPSgRCca4Dgtg3AdFr4nye4pIMMZ9WETT2d1D9fFWurp7gi5FRCRhTfiw6O5xHGlqp665Iy73f/z4cX72s58N+3ZXX301x48fj0NFIiLDN+HDIi05TE5aMnVN7XT3jPzeHkOFRXf3qTcwe+aZZ5g0adKI1yMicjomfFgAFGSn0tXjONYy8r2Lu+66i927d3P++edz0UUXcemll3LzzTezePFiAD72sY+xZMkSFi5cyNq1a/tuV1JSwpEjR6ioqGD+/Pl84QtfYOHChVxxxRW0traOeJ0iIqcyrk+djfTt3+/kreqGIa9v6+ymx0FGSjjm+1wwI4dvfXThKdt897vfZceOHWzfvp3169fz4Q9/mB07dvSd4vrQQw8xZcoUWltbueiii7juuuvIy8s74T527drFY489xv33388NN9zAb37zG265RbtlisjoUc/ClxwO4ZyjKw5DUZGWLVt2wmchfvzjH3PeeeexfPlyDhw4wK5du066TWlpKeeffz4AS5YsoaKiIq41iogMNGF6FtF6AM45dh1uAmBeYVbcTkXNzMzsu7x+/XpeeOEFXn31VTIyMli9evWgn5VITU3tuxwOhzUMJSKjTj0Ln5lRmJ1KW2c3jW1dI3a/2dnZNDY2DnpdfX09kydPJiMjg3feeYdNmzaN2OOKiIykCdOziEVuejIp4RCHG9vJTksakd5FXl4eK1euZNGiRaSnpzN16tS+66688kruvfdezj33XM4++2yWL19+xo8nIhIP5lx8x+hHy9KlS93AzY/efvtt5s+fP6z7qWtqp+p4K3MKsshKHVtZejq/r4hMbGa21Tm3NFo7DUMNMDkjhaRQiNrG9qBLERFJGAqLAUIhIz8rhca2Tlo7Tv3BORGRiUJhMYgpWSmEzdS7EBHxKSwGkRQKMSUrhfrWDtq71LsQEVFYDCE/KxXMOKLehYiIwmIoyeEQkzOSOdrSSaeWLxeRCU5hcQoFWangHHVNp9+7ON0lygF++MMf0tLSctqPLSIyUhQWp5CaHCYnPZm6pg66e06vd6GwEJHxYGx96iwAhdmp1Ld2UtfcQWF22rBvH7lE+eWXX05hYSFPPvkk7e3tfPzjH+fb3/42zc3N3HDDDVRWVtLd3c03vvENampqqK6u5tJLLyU/P5+XXnopDr+diEhsJk5YPHsXHHpz2DdLB+Z1dtPjHC4ljBGxBMi0xXDVd095+8glyp9//nmeeuopXn/9dZxzXHPNNbz88svU1tYyY8YMnn76acBbMyo3N5fvf//7vPTSS+Tn5w+7bhGRkaRhqBikhA3noKv7zJZGef7553n++ee54IILuPDCC3nnnXfYtWsXixcv5oUXXuDrX/86GzduJDc3d4QqFxEZGROnZxGlB3AqIec4VNtMV08PZ0/NPu0FBp1z3H333dxxxx0nXbd161aeeeYZ7r77bq644gq++c1vnna9IiIjTT2LGJgZBdmpdHT1UN/aOazbRi5R/qEPfYiHHnqIpiZv34yqqioOHz5MdXU1GRkZ3HLLLXzlK19h27ZtJ91WRCRIE6dncYZy0pJITQpT29hObnpyzL2LyCXKr7rqKm6++WZWrFgBQFZWFo8++ijl5eV89atfJRQKkZyczD333APA7bffzlVXXcX06dM1wS0igdIS5cNwtLmDymMtlOZnkp2WPGL3O1K0RLmIDJeWKI+DSRnJJPubI4mITCRxDQszu9LM3jWzcjO7a5DrLzGzbWbWZWbXD7juVjPb5X/dGs86YxUyIz8rleb2LprbR27rVRGRRBe3sDCzMPBT4CpgAfBJM1swoNl+4DbgVwNuOwX4FvA+YBnwLTObfDp1jPQw25TMFMKhxFu+fLwMJ4pIYopnz2IZUO6c2+Oc6wAeB66NbOCcq3DO/RUYuJbGh4A/OOeOOueOAX8ArhxuAWlpadTV1Y3oG2k4ZORlpdLQ1klbZ2IsX+6co66ujrS04X/CXEQkFvE8G6oIOBDxcyVeT+F0b1s0sJGZ3Q7cDjBr1qyT7qS4uJjKykpqa2tjfNjY9PQ4ahvaaDoUZnJmyoje9+lKS0ujuLg46DJEZJyKZ1gMdm5prH/ix3Rb59xaYC14Z0MNvD45OZnS0tIYH3J4fr1uJ49u2sfLX7uUGZPS4/IYIiKJIp7DUJXAzIifi4HqUbjtqPjCJXMAeGDj3oArERGJv3iGxWZgnpmVmlkKcBOwLsbbPgdcYWaT/YntK/xjCaNoUjrXnl/EY6/v51hzR9DliIjEVdzCwjnXBXwR703+beBJ59xOM/uOmV0DYGYXmVkl8AngPjPb6d/2KPB/4QXOZuA7/rGEcueqObR2dvPwqxVBlyIiElfj+hPco+ELj2xhc8VR/nzXB8hI0eopIjK26BPco2TN6jKOt3Ty2OsHojcWERmjFBZn6MJZk3lf6RQe2LiHjq7T23pVRCTRKSxGwJrVZRysb+M/t1cFXYqISFwoLEbAqrMKmD89h3s37KanZ3zMAYmIRFJYjAAzY83qMnbXNvOHt2uCLkdEZMQpLEbI1YumMWtKBj9bv1uL+onIuKOwGCFJ4RC3XzKHvxw4zqY9CfeREBGRM6KwGEHXLykmPyuVezbsDroUEZERpbAYQWnJYf7ub0p4+b1adlTVB12OiMiIUViMsFuWzyY7NYl71bsQkXFEYTHCctKS+dTy2Tzz5kEqjjQHXY6IyIhQWMTB360sISkcYu3GPUGXIiIyIhQWcVCYk8b1S4p5akslhxvagi5HROSMKSzi5I5L5tDV08NDr1QEXYqIyBlTWMTJ7LxMPnzuDB7dtI/61s6gyxEROSMKizi6c9Ucmtq7eHTTvqBLERE5IwqLOFo4I5dVZxXw81f20tbZHXQ5IiKnTWERZ2tWl3GkqYNfb60MuhQRkdOmsIiz95VO4YJZk1j78m66urU5koiMTQqLODMz1qwq48DRVp5+82DQ5YiInBaFxSj44PypzC3M4h4tXy4iY5TCYhSEQsadq8p451Aj69+rDbocEZFhU1iMkmvOm8GM3DTuWa8FBkVk7FFYjJKUpBCfv3gOr+89ytZ92hxJRMYWhcUoumnZTCZlJHPPei0wKCJji8JiFGWkJHHb+0t44e0a3qtpDLocEZGYKSxG2a0rSkhPDmtzJBEZUxQWo2xyZgqfXDaLddurqTzWEnQ5IiIxUVgE4PMXl2IGD2zcG3QpIiIxUVgEYMakdD52fhGPb95PXVN70OWIiESlsAjIHavm0N7Vw8N/rgi6FBGRqBQWAZlbmM0VC6by8Kv7aGrvCrocEZFTUlgE6M5VZdS3dvL46/uDLkVE5JQUFgG6YNZkVszJ4/6Ne2jv0uZIIpK44hoWZnalmb1rZuVmdtcg16ea2RP+9a+ZWYl/PNnMHjazN83sbTO7O551BmnN6jJqGtr5zzeqgy5FRGRIcQsLMwsDPwWuAhYAnzSzBQOafQ445pybC/wA+Ff/+CeAVOfcYmAJcEdvkIw3F8/LZ+GMHO59eTfdPVq+XEQSUzx7FsuAcufcHudcB/A4cO2ANtcCD/uXnwIuMzMDHJBpZklAOtABNMSx1sCYGWtWl7Gntpk/vHUo6HJERAYVz7AoAg5E/FzpHxu0jXOuC6gH8vCCoxk4COwHvuecO2mpVjO73cy2mNmW2tqxu0/EVYumMzsvQ5sjiUjCimdY2CDHBr4TDtVmGdANzABKgX80szknNXRurXNuqXNuaUFBwZnWG5hwyLjjkjL+UlnPq7vrgi5HROQk8QyLSmBmxM/FwMBZ3L42/pBTLnAUuBn4X865TufcYeAVYGkcaw3c315YREF2KvdogUERSUDxDIvNwDwzKzWzFOAmYN2ANuuAW/3L1wMvOm8cZj/wAfNkAsuBd+JYa+DSksN87m9K2bjrCG9W1gddjojICeIWFv4cxBeB54C3gSedczvN7Dtmdo3f7EEgz8zKgS8DvafX/hTIAnbghc7PnXN/jVetieJT75tFdlqSli8XkYSTFM87d849Azwz4Ng3Iy634Z0mO/B2TYMdH++y05L59PLZ3LNhN3uPNFOanxl0SSIigD7BnXA+u7KUlHCI+9S7EJEEorBIMAXZqdywdCa/2VbJofq2oMsREQEUFgnp9kvm0OPgoVe0OZKIJAaFRQKaOSWDj5w7nV9u2kd9S2fQ5YiIKCwS1Z2rymju6OYXmyqCLkVERGGRqOZPz+HSswv4+SsVtHZo+XIRCZbCIoGtWT2XuuYOfr31QPTGIiJxpLBIYBeVTGbJ7Mnct2EPnd09QZcjIhOYwiKBmRlrVpVRdbyVp/96MOhyRGQCU1gkuA+cU8hZU7O0fLmIBEphkeBCIePOVWW8W9PIS+8eDrocEZmgFBZjwEfPm0HRpHTuWa8lQEQkGAqLMSA5HOILF5eyueIYmytO2jBQRCTuFBZjxI0XzWJKZgr3qnchIgFQWIwR6Slhbnt/CX985zDvHGoIuhwRmWAUFmPIZ1bMJiMlrN6FiIy6mMLCzP7BzHL8bU4fNLNtZnZFvIuTE03KSOHmZbP4/V8PcuBoS9DliMgEEmvP4u+ccw3AFUAB8Fngu3GrSob0+YvnEDK4f+OeoEsRkQkk1rAw//vVePth/yXimIyiablp/O0FxTyx+QBHmtqDLkdEJohYw2KrmT2PFxbPmVk2oMWKAnL7qjl0dPfw769UBF2KiEwQsYbF54C7gIuccy1AMt5QlASgrCCLKxdO45FXK2hs0+ZIIhJ/sYbFCuBd59xxM7sF+CegPn5lSTR3riqjoa2Lx17fH3QpIjIBxBoW9wAtZnYe8DVgH/BI3KqSqM6bOYmVc/N4YONe2ru0OZKIxFesYdHlvCVPrwV+5Jz7EZAdv7IkFmtWzeVwYzu/3VYVdCkiMs7FGhaNZnY38GngaTML481bSIBWzs1jcVEu9728h+4eLV8uIvETa1jcCLTjfd7iEFAE/M+4VSUxMTPWrC5j75Fmntt5KOhyRGQciyks/ID4JZBrZh8B2pxzmrNIAB9aOI3S/ExtjiQicRXrch83AK8DnwBuAF4zs+vjWZjEJhwy7rhkDm9W1fNKeV3Q5YjIOBXrMNT/wPuMxa3Ouc8Ay4BvxK8sGY6PX1hEYXYqP1tfHnQpIjJOxRoWIedc5J6edcO4rcRZalKYz19cyp9317H9wPGgyxGRcSjWN/z/ZWbPmdltZnYb8DTwTPzKkuH65LJZ5KQlaflyEYmLWCe4vwqsBc4FzgPWOue+Hs/CZHiy05L5zIoSnnvrEOWHm4IuR0TGmZiHkpxzv3HOfdk599+cc7+NZ1Fyem5bWUJKOMTal9W7EJGRdcqwMLNGM2sY5KvRzLS3Z4LJz0rlpotm8ts3qjhY3xp0OSIyjpwyLJxz2c65nEG+sp1zOdHu3MyuNLN3zazczO4a5PpUM3vCv/41MyuJuO5cM3vVzHaa2ZtmlnY6v+BE8/mL59Dj4MGNe4MuRUTGkbid0eQvCfJT4CpgAfBJM1swoNnngGPOubnAD4B/9W+bBDwK3OmcWwisBrQWdwxmTsngmvNm8KvX93O8pSPockRknIjn6a/LgHLn3B7nXAfwON5ChJGuBR72Lz8FXGZmhrd961/9HflwztU557S0aozuWDWHlo5uHnl1X9CliMg4Ec+wKAIORPxc6R8btI1zrgtvj4w84CzA+afrbjOzrw32AGZ2u5ltMbMttbW1I/4LjFXnTMvhsnMK+fkre2np6Aq6HBEZB+IZFoPt0T1w8aKh2iQBfwN8yv/+cTO77KSGzq11zi11zi0tKCg403rHlTWryzjW0smTmw9EbywiEkU8w6ISmBnxczFQPVQbf54iFzjqH9/gnDvib+P6DHBhHGsdd5aWTOGiksncv3Evnd3aLl1Ezkw8w2IzMM/MSs0sBbgJWDegzTrgVv/y9cCL/iZLzwHnmlmGHyKrgLfiWOu4tGZ1GVXHW/n9XwZmtIjI8MQtLPw5iC/ivfG/DTzpnNtpZt8xs2v8Zg8CeWZWDnwZuMu/7THg+3iBsx3Y5px7Ol61jleXnl3I2VOzuXfDbnq0OZKInAEbL3sgLF261G3ZsiXoMhLO796o4ktPbOf+zyzl8gVTgy5HRBKMmW11zi2N1k4rx45zHzl3OsWT0/nZ+nJtjiQip01hMc4lhUPcfskc3th/nNf3Hg26HBEZoxQWE8AnlswkLzOFezZogUEROT0KiwkgPSXMZ1eWsP7dWt6q1vqPIjJ8CosJ4tPLS8hMCXOvehcichoUFhNEbkYyn1o+m//6azX761qCLkdExhiFxQTyub8pJSkUYu1G9S5EZHgUFhPI1Jw0rltSxJNbKqltbA+6HBEZQxQWE8ztl5TR2d3Dz1/R5kgiEjuFxQRTmp/J1Yum84tX99HQpv2kRCQ2CosJ6M5VZTS2d/Gr1/YHXYqIjBEKiwlocXEuF8/L58E/7aWtUxsQikh0CosJas2qMmob2/mPbVVBlyIiY4DCYoJaUZbHecW53Pfybrq0OZKIRKGwmKDMjDWry9hX18KzOw4FXY6IJDiFxQR2xYJpzCnI5J71u7V8uYicksJiAguFjDsvKeOtgw28vOtI0OWISAJTWExw114wg2k5adyzvjzoUkQkgSksJrjUpDCfv7iUTXuOsm3/saDLEZEEpbAQblo2i9z0ZO5drwUGRWRwCgshKzWJW1fM5vm3aig/3Bh0OSKSgBQWAsCt7y8hLTnEvRv2BF2KiCQghYUAkJeVyk0XzeJ3b1RRfbw16HJEJMEoLKTP5y8uBeCBjVq+XEROpLCQPsWTM7jm/Bk89vp+jjV3BF2OiCQQhYWc4M5VZbR2dvPwqxVBlyIiCURhISc4a2o2H5w/lX//cwUtHV1BlyMiCUJhISdZs7qM4y2dPPb6gaBLEZEEobCQkyyZPZllpVN4YOMeOrq0fLmIKCxkCGtWl3Gwvo3/3K7NkUQEkoIuQBLT6rMKOGdaNvdu2M11FxYTClnQJcl41nIUdr8I5X+EPevB9UDOdMie4X+fDjkz/O9F3rHU7KCrnlAUFjKo3s2R/uHx7fzh7Ro+tHBa0CXJeNLTDVXboPwF76tqK+AgfTLMuRRSMqDhIBzbC/tegbbjJ99HSvaJQdIXJhHfMwsgFB71X288UljIkD68eDrfe/5dfrZ+N1csmIqZehdyBhprYPcfvXDY/SK0HgMMipfC6rtg7gdhxgWDv7l3tEDjQWioHvz73peh8RC47hNvZ2HInjZ4kER+T8kYladgLFNYyJCSwiFuv6SMb/xuB5v2HGVFWV7QJclY0t0JB17r7z0cetM7nlkIZ10Fcy+Dsg9AxpTo95WSAXll3tdQerqhuXbwQGmohtp3YPdL0DHIYplpkyICJGL4K6eoP1Qy8mAC/8GksJBT+sSSYn70wnvcs2G3wkKiO77fm3cofwH2bPDemENJMHM5XPYtr/cwdRGE4nBuTcjvRWRHGTJtb/SGuBqr+4Ok8WD/sZqd0FQDDNhqOJzi3/+MAcNeA+ZWklJH/ndLAHENCzO7EvgREAYecM59d8D1qcAjwBKgDrjROVcRcf0s4C3gn51z34tnrTK4tOQwn11Zyv987l12VNWzqCg36JIkkXS2eXMKvQFx5F3veO5MWHy9Fw6ll0BaTrB1RkrNhoJsKDhr6DbdnV5g9IVKxPeGaji4Hd59FroGWXQzI39AgER+9y+nTRpzvZS4hYWZhYGfApcDlcBmM1vnnHsrotnngGPOublmdhPwr8CNEdf/AHg2XjVKbG5ZPpt71u/m3g27+cnNFwZdjgTJOTi6B3b9wQuHij95b5jhVChZCUtu9QIi/6wx92Z4gnAy5BZ7X0Nxzpt4HxgkkeFStRVaBtnfPil9kLO9ik48ljUNwokz+BPPSpYB5c65PQBm9jhwLV5Pode1wD/7l58CfmJm5pxzZvYxYA/QHMcaJQa56cncsnw2a1/eTcWRZkryM4MuSUZTexNUbOyfezhW4R2fUtYfDrNXTrxJYjPv7K30yTB1wdDtutq9yfcTgiRiTuXAa9713QMX7zTIKvSHuwY7hdjvqYzSKcTxDIsiIHK9iErgfUO1cc51mVk9kGdmrcDX8XolXxnqAczsduB2gFmzZo1c5XKSv1tZwkOv7GXtxj38y8cXB12OxJNzcPhtKPd7D/tehZ5OSM70hpRWfNGbnJ4yJ+hKx4akVJg82/sainPQUjf05Hy0U4jP/QR85Afx+x2Ib1gM1gd1Mbb5NvAD51zTqU7XdM6tBdYCLF26dOB9ywgqzEnj+iXFPLWlki9dNo/CnLSgS5KR1Hrc+zBc+Qve/ENjtXe8cCEsX+P1HmYtH7eTt4Ezg8x872v6uUO36z2FuHdCvqHKu1xwTtxLjGdYVAIzI34uBqqHaFNpZklALnAUrwdyvZn9f8AkoMfM2pxzP4ljvRLF7RfP4fHX9/PQKxXcdVX8/3NKHPX0eJO0vRPTlZu9zyik5kLZai8cyi6D3KKgK5VIsZxCHCfxDIvNwDwzKwWqgJuAmwe0WQfcCrwKXA+86JxzwMW9Dczsn4EmBUXwSvIzuXrxdB7dtI+bLprJ7LwMfVBvLGk+4i+p4fceeideZ1wAF3/ZC4iipQk1qSqJI27/K/w5iC8Cz+GdOvuQc26nmX0H2OKcWwc8CPzCzMrxehQ3xaseGRl3rirj2R2HWP299UzJTGHhjBwWFeWy2P8qnpyuAEkU3V1QtaV/Yrp6O+C8D5eVXeb3Hj4AWQVBVypjgHl/yI99S5cudVu2bAm6jAnhvZpGXttTx5tV9eyoauC9mka6erz/R7npySwqymHRjFwWFXlfs6dkaCHC0dJQHfGhuJegrR4sBMXLvHCYexlMPz8+H4qTMcnMtjrnlkZrp/6mDNtZU7M5a2r/6Xptnd28V9PYFx47qur5+SsVdHR7e2Fkpyax0A+QxcW5LJyRy5z8TAXISOhqh/2b+oeWDu/0jmdPh/kfhbmXw5xV3umdImdAYSFnLC05zLnFkzi3eFLfsY6uHt6raWRHVT07qut5s6qBRzbt69tMKTMlzAJ/CKs3RMoKsggrQKI7VtEfDns2QGczhJJh9gq4/DteD6Jwwdj+UJwkHIXFkV3w69v809YKvQ/BZBb0f4+8HE4OutoxIyUp1DcM1auzu4fyw01egFTVs6O6gcdfP0BrZwUAackhFkzPYXFRLgv9OZC5hVkkhyf4kElHi7+khj/3UFfuHZ80C867CeZdDiUXQ2pWsHXKuKY5i9r34IVvQdNhaD4MTbWDr/cC3nouWYV+qBR43zML+i9nFfaHzkT7NOtp6u5x7Klt4s2qet6sqmdnVQM7q+tp7vCWmk5JCjF/eg6LZnghsqgol7OmZpOSNI4DxDnvj5jecNj3CnS1QVKaFwpzP+h95ZWp9yBnLNY5C4XFQM5BR5O31HFTrR8gh73TDvsu1/Yfa68f/H5Ssk7umZzUW/FDJzVHL/oIPT2OvXXNfT2Q3hBpbO8CICXsYsDEAAAPTElEQVQc4uxp2d5Euj+Mdfa0bNKSx/AmN20N3p4MvcNL9fu94/ln909Mz34/JKcHW6eMOwqL0dLZ5oVHsx8ekT2U5sMnhk7LUU7+EDveImwn9FD8QIkcFusNmPQpE/JMlp4ex/6jLd4kerU/jFXVQH1rJwBJIWPe1GwW9wZIUS7zp+WQnpKgAeIc1OzoD4f9r0JPl/dHxpzV/l4Pl516iQiREaCwSETdXd76Lyf1UGojLvf2Ymq9N4+BLNw/1JWZP0ivJXKILH9cz7M456g81trX+9hR7Z2JdbTZW5AtHDLmFmT54eENY82fnkNmakBTdS1HvdNZe09tbarxjk9b3D+0VLwMklKCqU8mJIXFWNfT4y0a1hcmh4cYFvOPdbUNfj/pkwfpoQzRaxkHQxzOOarr2/on0au8M7GONLUD3mhfWUEWi6Zns3hGFoumZ7JgajrZyXh7GPR0+t+7In7uijg+8Oeu6LfrbPFOb63aCq7Hm/sq+0D/8FK0zXpE4kifsxjrQiFvu8mMKUCUdZh651lO6K0MMix28C/e9e0Ng99PSnbEEFjEvEpfD8YPmN7tJc/0TfSMbjd4O+vppKi7i6KeTj7kH3dpnbjkTrq7Orw2DV0kNXTBuyP+rza4cApMOxcu+Zq/pMaFg+8zLZLAFBbjgZm3pn1qdmwLjHW2Rsyl1J44LNYbNnXlsO/P0Ho0/vUPJpTsDaGFkr21ivp+Thr6eHL6oO0snISFkgkNON7UZdQ0dXOwsYvKhk4O1HdxpLWHLhemkzCTsjIoysuhOD+bWQWTKCmYRHZmepSaBvwcCuvkBRkXFBYTUXK6d47+pBj2AOnu9OZZInsovQvQnfSmOcSbaDglId9gs/yvyHita2pnZ3WDdwZWdT0vVdVzYG/vqdQtFE1y/im8GX0T6flZWrZbxj/NWYhEcbyloy9AeudBKupa+q6fnpvGwhm5foh4E+na70PGCs1ZiIyQSRkprJybz8q5+X3HGto6+z5A2Bsif3ynht6/vQqyU/s+RLhoRg7nTMuheHK61sOSMUthIXIactKSWVGWx4qyvL5jTe1dvOWfvtu7Jtb6dw/jL8hLWnKIuYVZzCvM9r9ncdbUbGZOydCaWJLwFBYiIyQrNYllpVNYVjql71hLRxdvH2xkV00juw43setwE5v21PHbN6r62qQkhSgr8MJjXmEW86ZmM29qFrOnZJA00dfFkoShsBCJo4yUJJbMnsyS2ScuEd7Y1km5Hx7lh5vYVdPI1n3HWPeX/p2Hk8PGnPws5k7t74XMK8xidl7m+F4bSxKSwkIkANlpyVwwazIXzDoxRJrbu9hd28Sumt4gaeTNynqeefNg33xIUsgoyc88qSdSmp9JapI+vyHxobAQSSCZqUkn7Q0C0NrRze5avxdyuJFdNU28c6iR53Ye6psTCRmU5GUyt7cXMjWLuYVZlBVkje1FFiUhKCxExoD0lPBJ+4OAt0vh3iPNXi/Enxd5r6aRP75zmG4/Rcxg1pSM/l6IP8leVphJRoreAiQ2+p8iMoalJYeZPz2H+dNzTjje0dVDRV2zP5zlT67XNLLhvVo6u/s/W1U8Ob1vLmSuHyZzC7PICmqxRUlY+h8hMg6lJIUi9kqf3ne8s7uHfXUtlPtDWb09kT/tOtK3ZzrAjNy0/l7I1CzmFnrDWjlp43cVYzk1hYXIBJIc9j7rMbcwiysX9R/v6u7hwLHWvlN8y/0Q2bSnjvau/hCZlpPWNxcyzw+QeYVZTMrQsurjncJCREgKhyjNz6Q0P5MrFvYf7+5xVB1rjRjK8oa1nth8gBZ/61vwPrHee3bW3L55kSzytG7WuKGwEJEhhUPGrLwMZuVlcNn8qX3He3oc1fWt/sS6FyDv1TTxm21VNLX3b9qVl5niz4X4PRF/XiQ/KwXTarxjisJCRIYtFDKKJ2dQPDmDS88u7DvunONQQ9sJnxN5r6aJdduraWjrD5FJGcn+pHr/vMhZU7MpzE5ViCQohYWIjBgzY3puOtNz07nkrIK+4845ahvb+87K6h3SenbHQR5r6exrl52WxLzCLGZOyWBqTpr/lco0/3JhTqo+eBgQhYWIxJ2ZUZiTRmFO2gmr9zrnqGvuYFeN1wvpDZE39h+npqHthMn1XpMzkvuCZJofJlNz05ianca0XC9Q8jNTtcLvCFNYiEhgzIz8rFTys1JPWMEXvCCpb+2kpqGdQw1t1DS0UVPfRk1jG4fq2znc2MbbB7391XsGbMuTFDIKslNP6JkU9oVLGtNyUynMSSM7NUnDXjFSWIhIQjIzJmWkMCkjhbOnZQ/Zrqu7hyNNHdQ0tHGooY3D/veahnZqGtrYe6SZV3fXnTBn0isjJdwXKL09lcLIHouGvvooLERkTEsKh5iW6w1BnXeKdq0d3V7vpC9UInosDW28sf84hxra6Bhk6GtKZgqF2alM84e7puaePJcy3oe+FBYiMiGkp4Qpyc+kJD9zyDa9Q199PZP6/nDp7am8VT0xh74UFiIivsihr3OmDd2ud+grsmdS09A/l7KndvwNfSksRESGKXLo61RaOrpOGu7qnbA/3NDGtv3HqGloP62hr6k5aeRlpoza0JfCQkQkTjJSkijJT4o69HW8pdM/y+vkuZSahnZ2+kNfbpChr8LsVK5ePJ1/+siCuP4ucQ0LM7sS+BEQBh5wzn13wPWpwCPAEqAOuNE5V2FmlwPfBVKADuCrzrkX41mriEgQzIzJmSlMzkzhnGk5Q7br6u6htqnd65nUt3HYD5eahnZmTEqPe51xCwszCwM/BS4HKoHNZrbOOfdWRLPPAcecc3PN7CbgX4EbgSPAR51z1Wa2CHgOKIpXrSIiiS4pHOr7dDwzR//x47nr+zKg3Dm3xznXATwOXDugzbXAw/7lp4DLzMycc28453p3rt8JpPm9EBERCUA8w6IIOBDxcyUn9w762jjnuoB6IG9Am+uAN5xz7QMfwMxuN7MtZraltrZ2xAoXEZETxTMsBpuid8NpY2YL8Yam7hjsAZxza51zS51zSwsKCgZrIiIiIyCeYVHJiSNrxUD1UG3MLAnIBY76PxcDvwU+45zbHcc6RUQkiniGxWZgnpmVmlkKcBOwbkCbdcCt/uXrgRedc87MJgFPA3c7516JY40iIhKDuIWFPwfxRbwzmd4GnnTO7TSz75jZNX6zB4E8MysHvgzc5R//IjAX+IaZbfe/ChERkUCYG/gpjzFq6dKlbsuWLUGXISIyppjZVufc0mjt4jkMJSIi48S46VmYWS2w7wzuIh/vw4CJRnUNj+oaHtU1POOxrtnOuaink46bsDhTZrYllq7YaFNdw6O6hkd1Dc9ErkvDUCIiEpXCQkREolJY9FsbdAFDUF3Do7qGR3UNz4StS3MWIiISlXoWIiISlcJCRESimlBhYWZXmtm7ZlZuZncNcn2qmT3hX/+amZUkSF23mVltxNInnx+luh4ys8NmtmOI683MfuzX/VczuzBB6lptZvURz9c3R6mumWb2kpm9bWY7zewfBmkz6s9ZjHWN+nNmZmlm9rqZ/cWv69uDtBn112SMdQX1mgyb2Rtm9l+DXBff58o5NyG+8LZ23Q3Mwduu9S/AggFt/g/gXv/yTcATCVLXbcBPAnjOLgEuBHYMcf3VwLN4S80vB15LkLpWA/8VwPM1HbjQv5wNvDfIv+WoP2cx1jXqz5n/HGT5l5OB14DlA9oE8ZqMpa6gXpNfBn412L9VvJ+ridSzOO2d+xKgrkA4517GXzJ+CNcCjzjPJmCSmU1PgLoC4Zw76Jzb5l9uxFtAc+CGX6P+nMVY16jzn4Mm/8dk/2vgGTej/pqMsa5R52/b8GHggSGaxPW5mkhhMVI79wVRF8B1/rDFU2YWwA68g4q19iCs8IcRnvU30RpV/hDABXh/lUYK9Dk7RV0QwHPmD6tsBw4Df3DODfl8jeJrMpa6YPRfkz8Evgb0DHF9XJ+riRQWZ7xzX5zE8pi/B0qcc+cCL9D/10PQgni+YrENb72b84B/A343mg9uZlnAb4AvOecaBl49yE1G5TmLUlcgz5lzrts5dz7e5mjLzGzRgCaBPF8x1DWqr0kz+whw2Dm39VTNBjk2Ys/VRAqLM9q5L8i6nHN1rn8P8vuBJXGuKVaxPKejzjnX0DuM4Jx7Bkg2s/zReGwzS8Z7Q/6lc+4/BmkSyHMWra4gnzP/MY8D64ErB1wVxGsyal0BvCZXAteYWQXeUPUHzOzRAW3i+lxNpLA47Z37gq5rwJj2NXhjzolgHfAZ/wyf5UC9c+5g0EWZ2bTesVozW4b3/7xuFB7X8Db0ets59/0hmo36cxZLXUE8Z2ZWYN6umJhZOvBB4J0BzUb9NRlLXaP9mnTO3e2cK3bOleC9R7zonLtlQLO4PldJI3VHic4512VmvTv3hYGHnL9zH7DFObcO7wX1C/N27juK94+SCHX9vXm7C3b5dd0W77oAzOwxvLNk8s2sEvgW3mQfzrl7gWfwzu4pB1qAzyZIXdcDa8ysC2gFbhqF0Afvr79PA2/6490A/x2YFVFbEM9ZLHUF8ZxNBx42szBeOD3pnPuvoF+TMdYVyGtyoNF8rrTch4iIRDWRhqFEROQ0KSxERCQqhYWIiESlsBARkagUFiIiEpXCQiQBmLfq60kriYokCoWFiIhEpbAQGQYzu8Xf62C7md3nLzjXZGb/v5ltM7M/mlmB3/Z8M9vkLzb3WzOb7B+fa2Yv+Iv2bTOzMv/us/xF6d4xs1+OworHIjFTWIjEyMzmAzcCK/1F5rqBTwGZwDbn3IXABrxPlAM8AnzdX2zuzYjjvwR+6i/a936gd7mPC4AvAQvw9jdZGfdfSiRGE2a5D5ERcBnegnGb/T/60/GWsO4BnvDbPAr8h5nlApOccxv84w8DvzazbKDIOfdbAOdcG4B/f6875yr9n7cDJcCf4v9riUSnsBCJnQEPO+fuPuGg2TcGtDvVGjqnGlpqj7jcjV6fkkA0DCUSuz8C15tZIYCZTTGz2Xivo+v9NjcDf3LO1QPHzOxi//ingQ3+PhKVZvYx/z5SzSxjVH8LkdOgv1xEYuSce8vM/gl43sxCQCfwfwLNwEIz24q3O9mN/k1uBe71w2AP/SvMfhq4z18xtBP4xCj+GiKnRavOipwhM2tyzmUFXYdIPGkYSkREolLPQkREolLPQkREolJYiIhIVAoLERGJSmEhIiJRKSxERCSq/w1r9wbMJfDE4gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1be4fb905c0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(hist.history['loss'])\n",
    "plt.plot(hist.history['val_loss'])\n",
    "plt.title('model loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'test'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Percentage accuracy of model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 98.91%\n"
     ]
    }
   ],
   "source": [
    "labels_pred = model.predict(data_test)\n",
    "labels_pred = np.round(labels_pred.flatten())\n",
    "accuracy = accuracy_score(labels_test, labels_pred)\n",
    "print(\"Accuracy: %.2f%%\" % (accuracy*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "f1, precision, and recall scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      2382\n",
      "          1       0.98      0.96      0.97       462\n",
      "\n",
      "avg / total       0.99      0.99      0.99      2844\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(labels_test, labels_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Section 5: Comparing the Model to Base Line\n",
    "\n",
    "In order to evaluate the effectiveness of the LSTM + CNN model, a logistic regression model is trained with the same train data and the same number of epochs, and tested with the same test data."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression Base Line Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LogReg:\n",
    "    \"\"\"\n",
    "    Class to represent a logistic regression model.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, l_rate, epochs, n_features):\n",
    "        \"\"\"\n",
    "        Create a new model with certain parameters.\n",
    "\n",
    "        :param l_rate: Initial learning rate for model.\n",
    "        :param epoch: Number of epochs to train for.\n",
    "        :param n_features: Number of features.\n",
    "        \"\"\"\n",
    "        self.l_rate = l_rate\n",
    "        self.epochs = epochs\n",
    "        self.coef = [0.0] * n_features\n",
    "        self.bias = 0.0\n",
    "\n",
    "    def sigmoid(self, score, threshold=20.0):\n",
    "        \"\"\"\n",
    "        Prevent overflow of exp by capping activation at 20.\n",
    "\n",
    "        :param score: A real valued number to convert into a number between 0 and 1\n",
    "        \"\"\"\n",
    "        if abs(score) > threshold:\n",
    "            score = threshold * sign(score)\n",
    "        activation = exp(score)\n",
    "        return activation / (1.0 + activation)\n",
    "\n",
    "    def predict(self, features):\n",
    "        \"\"\"\n",
    "        Given an example's features and the coefficients, predicts the class.\n",
    "\n",
    "        :param features: List of real valued features for a single training example.\n",
    "\n",
    "        :return: Returns the predicted class (either 0 or 1).\n",
    "        \"\"\"\n",
    "        value = sum([features[i]*self.coef[i] for i in range(len(features))]) + self.bias\n",
    "        return self.sigmoid(value)\n",
    "\n",
    "    def sg_update(self, features, label):\n",
    "        \"\"\"\n",
    "        Computes the update to the weights based on a predicted example.\n",
    "\n",
    "        :param features: Features to train on.\n",
    "        :param label: Corresponding label for features.\n",
    "        \"\"\"\n",
    "        yhat = self.predict(features)\n",
    "        e = label - yhat\n",
    "        self.bias = self.bias + self.l_rate * e * yhat * (1-yhat)\n",
    "        for i in range(len(features)):\n",
    "            self.coef[i] = self.coef[i] + self.l_rate * e * yhat * (1-yhat) * features[i]\n",
    "        return\n",
    "\n",
    "    def train(self, X, y):\n",
    "        \"\"\"\n",
    "        Computes logistic regression coefficients using stochastic gradient descent.\n",
    "\n",
    "        :param X: Features to train on.\n",
    "        :param y: Corresponding label for each set of features.\n",
    "\n",
    "        :return: Returns a list of model weight coefficients where coef[0] is the bias.\n",
    "        \"\"\"\n",
    "        for epoch in range(self.epochs):\n",
    "            for features, label in zip(X, y):\n",
    "                self.sg_update(features, label)\n",
    "        return self.bias, self.coef"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_accuracy(y_bar, y_pred):\n",
    "    \"\"\"\n",
    "    Computes what percent of the total testing data the model classified correctly.\n",
    "\n",
    "    :param y_bar: List of ground truth classes for each example.\n",
    "    :param y_pred: List of model predicted class for each example.\n",
    "\n",
    "    :return: Returns a real number between 0 and 1 for the model accuracy.\n",
    "    \"\"\"\n",
    "    correct = 0\n",
    "    for i in range(len(y_bar)):\n",
    "        if y_bar[i] == y_pred[i]:\n",
    "            correct += 1\n",
    "    accuracy = (correct / len(y_bar)) * 100.0\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training the logistic regression model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Logistic Model\n",
    "logreg = LogReg(LEARNING_RATE, EPOCHS, len(data_train[0]))\n",
    "bias_logreg, weights_logreg = logreg.train(data_train, labels_train)\n",
    "y_logistic = [round(logreg.predict(example)) for example in data_test]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Getting the accuracy of the logistic regression model predicting the test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Accuracy: 83.755\n"
     ]
    }
   ],
   "source": [
    "# Compare accuracies\n",
    "accuracy_logistic = get_accuracy(y_logistic, labels_test)\n",
    "print('Logistic Regression Accuracy: {:0.3f}'.format(accuracy_logistic))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}