LDA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "np.set_printoptions(precision=3, suppress=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class LDA:\n",
    "    labels = None\n",
    "    coeff = None\n",
    "    eig_vecs, eig_vals = None, None\n",
    "    classifier = None\n",
    "    \n",
    "    def fit(self, X, y):\n",
    "        # Implemented from Pattern Classification by Richard O. Duda, \n",
    "        # Peter E. Hart  and DAvid G. Stork pg 121 Multiple Discriminant Analysis\n",
    "        self.labels = np.unique(y).astype(int)\n",
    "        num_classes = self.labels.size\n",
    "        num_feat = X.shape[1]\n",
    "        \n",
    "        m = X.mean(axis=0)\n",
    "        \n",
    "        St = np.cov((X - m).T)\n",
    "        Sb = np.empty((num_classes, num_feat, num_feat))\n",
    "        mi = np.empty((num_classes, num_feat))\n",
    "        \n",
    "        ni = np.bincount(y)[self.labels]\n",
    "        \n",
    "        for index, label in enumerate(self.labels):\n",
    "            mi[index] = X[y == label].mean(axis=0)\n",
    "            Sb[index] = ni[index] * np.cov(X[y == label].mean(axis=0).T - m.T)\n",
    "        Sb = Sb.sum(axis=0)\n",
    "        Sw = St - Sb\n",
    "        w = np.linalg.pinv(Sw) @ (mi[0] - mi[1]).T\n",
    "        self.coeff = w\n",
    "        self.eig_vals, self.eig_vecs = np.linalg.eigh(np.linalg.pinv(Sw) @ Sb)\n",
    "        X_recon = np.dot(self.coeff, X.T).reshape(-1, 1)\n",
    "        self.classifier = GaussianNB()\n",
    "        self.classifier.fit(X_recon, y)\n",
    "        \n",
    "    def predict(self, X):\n",
    "        X_recon = np.dot(self.coeff, X.T).reshape(-1, 1)\n",
    "        return self.classifier.predict(X_recon)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def sample_mask_from(bool_array, sample_size):\n",
    "    sample_rows_mask = bool_array.copy()\n",
    "    temp = bool_array[bool_array == True]\n",
    "    temp[sample_size:] = False\n",
    "    np.random.shuffle(temp)\n",
    "    sample_rows_mask[sample_rows_mask == True] = temp\n",
    "    return sample_rows_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "wine_data = np.genfromtxt('../Data/wine.data', delimiter=',')\n",
    "rows_1 = (wine_data[:,0] == 1)\n",
    "rows_2 = (wine_data[:,0] == 2)\n",
    "wine_data = wine_data[rows_1 + rows_2]\n",
    "rows_1 = (wine_data[:,0] == 1)\n",
    "rows_2 = (wine_data[:,0] == 2)\n",
    "five_sample_rows_mask = sample_mask_from(rows_1, 5) + sample_mask_from(rows_2, 5)\n",
    "train_set_1 = wine_data[five_sample_rows_mask]\n",
    "test_set_1 = wine_data[~five_sample_rows_mask]\n",
    "\n",
    "fifty_sample_rows_mask = sample_mask_from(rows_1, 50) + sample_mask_from(rows_2, 50)\n",
    "train_set_2 = wine_data[fifty_sample_rows_mask]\n",
    "test_set_2 = wine_data[~fifty_sample_rows_mask]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fit data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "lda_1, lda_2 = LDA(), LDA()\n",
    "lda_1.fit(train_set_1[:, 1:], train_set_1[:, 0].astype(int))\n",
    "lda_2.fit(train_set_2[:, 1:], train_set_2[:, 0].astype(int))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predict labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "w_pred_lab_1, w_pred_lab_2 = lda_1.predict(test_set_1[:, 1:]), lda_2.predict(test_set_2[:, 1:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix, accuracy_score, classification_report\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluate model performance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Wine Data:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For five samples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class 1</th>\n",
       "      <th>Class 2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Class 1</th>\n",
       "      <td>42</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class 2</th>\n",
       "      <td>10</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Class 1  Class 2\n",
       "Class 1       42       12\n",
       "Class 2       10       56"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(confusion_matrix(test_set_1[:, 0], w_pred_lab_1), columns=['Class 1', 'Class 2'], index = ['Class 1', 'Class 2'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For fifty samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class 1</th>\n",
       "      <th>Class 2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Class 1</th>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class 2</th>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Class 1  Class 2\n",
       "Class 1        9        0\n",
       "Class 2        0       21"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(confusion_matrix(test_set_2[:, 0], w_pred_lab_2), columns=['Class 1', 'Class 2'], index = ['Class 1', 'Class 2'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Accuracy Score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For five samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy is: 81.7%\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy is: {:.3}%\".format(accuracy_score(test_set_1[:, 0], w_pred_lab_1, normalize=True) * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For fifty samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "    Class 1       0.81      0.78      0.79        54\n",
      "    Class 2       0.82      0.85      0.84        66\n",
      "\n",
      "avg / total       0.82      0.82      0.82       120\n",
      "\n"
     ]
    }
   ],
   "source": [
    "target_names = ['Class 1', 'Class 2']\n",
    "print(classification_report(test_set_1[:, 0], w_pred_lab_1, target_names=target_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For fifty samples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class 1</th>\n",
       "      <th>Class 2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Class 1</th>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class 2</th>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Class 1  Class 2\n",
       "Class 1        9        0\n",
       "Class 2        0       21"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(confusion_matrix(test_set_2[:, 0], w_pred_lab_2), columns=['Class 1', 'Class 2'], index = ['Class 1', 'Class 2'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Accuracy Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy is: 100.0%\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy is: {:.4}%\".format(accuracy_score(test_set_2[:, 0], w_pred_lab_2, normalize=True) * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "    Class 1       1.00      1.00      1.00         9\n",
      "    Class 2       1.00      1.00      1.00        21\n",
      "\n",
      "avg / total       1.00      1.00      1.00        30\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(test_set_2[:, 0], w_pred_lab_2, target_names=target_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "K fold cross validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import StratifiedKFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_folds = 10\n",
    "skf = StratifiedKFold(wine_data[:, 0], n_folds=num_folds, shuffle=True)\n",
    "X = wine_data[:, 1:]\n",
    "y = wine_data[:, 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "accuracy_scores = []\n",
    "for train_index, test_index in skf:\n",
    "    X_train, X_test = X[train_index], X[test_index]\n",
    "    y_train, y_test = y[train_index], y[test_index]\n",
    "    clf = LDA()\n",
    "    clf.fit(X_train, y_train.astype(int))\n",
    "    y_pred = clf.predict(X_test)\n",
    "    accuracy_scores.append(accuracy_score(y_test.astype(int), y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "Average accuracy from cross validation is"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy is: 98.46 %\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy is: {:.4} %\".format(np.asarray(accuracy_scores).mean()*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNIST Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load MNIST Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mnist_data = np.genfromtxt('../Data/MNIST_HW1/train.csv', delimiter=',', dtype=np.float)\n",
    "mnist_data = mnist_data.T\n",
    "X = mnist_data[:, :-1]\n",
    "y = mnist_data[:, -1].astype(int)\n",
    "# Select data corresponding to 0 and 1 digits\n",
    "rows_0_1 = ((y == 0) + (y == 1))\n",
    "# Select data corresponding to 3 and 5 digits\n",
    "rows_3_5 = ((y == 3) + (y == 5))\n",
    "X_0_1 = X[rows_0_1]\n",
    "X_3_5 = X[rows_3_5]\n",
    "y_0_1 = y[rows_0_1].astype(int)\n",
    "y_3_5 = y[rows_3_5].astype(int)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fit to model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lda_01, lda_35 = LDA(), LDA()\n",
    "lda_01.fit(X_0_1, y_0_1)\n",
    "lda_35.fit(X_3_5, y_3_5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load MNIST Test Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mnist_test_data = np.genfromtxt('../Data/MNIST_HW1/test.csv', delimiter=',', dtype=np.float)\n",
    "mnist_test_data = mnist_test_data.T\n",
    "X_test = mnist_test_data[:, :-1]\n",
    "y_test = mnist_test_data[:, -1].astype(int)\n",
    "# Select data corresponding to 0 and 1 digits\n",
    "rows_0_1 = ((y_test == 0) + (y_test == 1))\n",
    "# Select data corresponding to 3 and 5 digits\n",
    "rows_3_5 = ((y_test == 3) + (y_test == 5))\n",
    "X_test_0_1 = X_test[rows_0_1]\n",
    "X_test_3_5 = X_test[rows_3_5]\n",
    "y_test_0_1 = y_test[rows_0_1]\n",
    "y_test_3_5 = y_test[rows_3_5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred_0_1 = lda_01.predict(X_test_0_1)\n",
    "y_pred_3_5 = lda_35.predict(X_test_3_5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluate model performance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Confusion matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "        0 1 labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Digit 0</th>\n",
       "      <th>Digit 1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Digit 0</th>\n",
       "      <td>976</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Digit 1</th>\n",
       "      <td>12</td>\n",
       "      <td>1123</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Digit 0  Digit 1\n",
       "Digit 0      976        4\n",
       "Digit 1       12     1123"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(confusion_matrix(y_test_0_1, y_pred_0_1), columns=['Digit 0', 'Digit 1'], index = ['Digit 0', 'Digit 1'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "        3 5 labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Digit 0</th>\n",
       "      <th>Digit 1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Digit 0</th>\n",
       "      <td>985</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Digit 1</th>\n",
       "      <td>41</td>\n",
       "      <td>851</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Digit 0  Digit 1\n",
       "Digit 0      985       25\n",
       "Digit 1       41      851"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(confusion_matrix(y_test_3_5, y_pred_3_5), columns=['Digit 0', 'Digit 1'], index = ['Digit 0', 'Digit 1'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Accuracy score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "        0 1 digits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy is: 99.24%\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy is: {:.4}%\".format(accuracy_score(y_test_0_1, y_pred_0_1, normalize=True) * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "        3 5 digits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy is: 96.53%\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy is: {:.4}%\".format(accuracy_score(y_test_3_5, y_pred_3_5, normalize=True) * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LDA for dimensionality reduction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cumulative Eigen Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1168016d8>]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEACAYAAAC9Gb03AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAF2lJREFUeJzt3X2QHdV55/HvT+8SEkK8jUBCsiWjgBxAgCPEah1d7IVY\nkEJ/hCrsdYyXqq0izjp2OVnWjssp5o+NY1fiSmGbKlm12AusWbuC11g2kCUOXNjgyhikmaAICSQQ\nthBoLFlvCBEs0LN/dI80Ht/RXM3c291z7u9T1TX9cqbPM3ek5555+nRfRQRmZpauCWUHYGZm7eVE\nb2aWOCd6M7PEOdGbmSXOid7MLHFO9GZmiWs60UuaIGmjpPXDHP+qpG2S+iQta12IZmY2Fqcyov80\n8FyjA5JWA4sj4kLgNmBtC2IzM7MWaCrRS5oPXA/8j2GarAHuBYiIHmC2pK6WRGhmZmPS7Ij+b4Hb\ngeFuo50H7By0vSvfZ2ZmJRsx0Uu6AeiPiD5A+WJmZuPEpCbarARulHQ9MB2YJeneiLhlUJtdwAWD\ntufn+36NJD9Yx8xsFCJi1IPsEUf0EfH5iFgQEYuADwOPDUnyAOuBWwAkrQAORET/MOer/HLHHXeU\nHoPjdJzjNUbH2fplrJoZ0Tck6bYsb8e6iHhY0vWStgNvALeOOTIzM2uJU0r0EfEE8ES+/o0hxz7Z\nwrjMzKxFfGdsA7VarewQmuI4W2s8xDkeYgTHWTVqRf2n6c6kaHd/v/wlbN0K27bB9u3Q3w/79mX7\nDxyAd97JlmPHsmVgfeBrO8IbL+dshk5yOWi8HKtaPFU6VrV4ij42axacd162LFoE73lPtixcCJMn\nD/997SaJGMPF2HGd6I8ehaefhieegGeegQ0bYP9+uPhiuPDC7Bc0dy6cdRaceSaccUb2y5owASZO\nzL4OXZ/Qpr9xRvrPVZVznszJfnXj5VjV4qnSsarFU8axQ4fgtddg1y546SV48cVs6e+HSy6B3/kd\nuPZauO46mDZt+HO1Wkcm+v374ctfhrvvhvnz4ZprYPlyuPJKWLy4fcnazDrT4cOwcSP09MBDD8Hz\nz8NnPgN/8icwfXr7+++4RP/AA9mLu2YN3H57ltjNzIq0aRPccQds3gz33ZcNNNupoxL9l74Ea9fC\nd74DK1a0MDAzs1F44AH44z+Gr30Nbr65ff2MNdGPeh590f7mb+Dee+EnP4Hzzy87GjMzuOkmWLLk\nRM1+zZqyI2psXIzof/Sj7F3zqafgggtGbm9mVqRnnoHVq+HHP4bLLmv9+cc6oq/8ZcsDB+CP/gju\nucdJ3syq6X3vgzvvzEb4r79edjS/qfIj+j/7s+yFW7euTUGZmbXIH/5hNuf+L/+ytedN+mLs3r1Z\n/WvTJpjnp9ubWcW98kpWutm4MUv4rZJ06ebrX4c/+AMneTMbH+bPh9tug7/+67Ij+XWVHdEfO5a9\nI/7938N739vmwMzMWmTXruwu2h07YPbs1pwz2RH9T3+aPXfCSd7MxpN582DVKvjBD8qO5ITKJvrv\nfS+7gm1mNt78/u/DI4+UHcUJlSzdRGSPNvj+99szJ9XMrJ127YJLL80ehjapBbelJlm6eeGF7MmU\nl15adiRmZqdu3rzsvp+enrIjyVQy0T/+OHzwg8U/htfMrFVWr65O+aaSif6xx+ADHyg7CjOz0bvm\nGnjyybKjyFSuRn/sGHR1ZTcc+JEHZjZevf569klVe/eO/UNKkqvR/+u/ZnNPneTNbDybNQuWLs2m\nipetcon+Jz+B97+/7CjMzMbud3+3GuWbyiX6nh646qqyozAzG7txk+glTZXUI6lX0mZJX2zQZpWk\nA5I25ssXRhtQT48/PcrM0nD11fD009m1xzKNOJU/It6SdE1EHJE0EXhK0sqIeGpI0ycj4saxBHPg\nAOzcCb/922M5i5lZNZxzDsyZA9u2wW/9VnlxNFW6iYgj+erU/Hv2N2g25lnvTz8Nl1/emjvJzMyq\nYPny8m+cairRS5ogqRfYDdQj4rkGza6W1CfpIUlLRxOM6/Nmlpqrrip/5k1TY+eIOAZcLul04FFJ\nqyLiiUFNNgAL8vLOauBBYEmjc3V3dx9fr9Vq1Gq149s9PfDxj5/qj2BmVl3Ll8P995/a99Trder1\nestiOOUbpiT9BXAkIr5ykjY7gCsjYt+Q/cPeMBWR3Si1YYPn0JtZOo4cgbPPhv37YerU0Z2j7TdM\nSTpb0ux8fTpwLdA3pE3XoPXlZG8gv5bkR/LyyzB5cvYJLWZmqZgxAxYtgucaFbwL0kzp5jzgHkki\ne2O4LyL+UdJtQETEOuAmSZ8AjgJvAjefaiA9PdmfOH6QmZmlZtky6OvLJpuUoZnplZuAKxrs/8ag\n9buAu8YSSG8vXHnlWM5gZlZNA4m+LJW5M7a3N3sxzMxSU3air8TTKwcuxPb2Zg/sNzNLyZ49cOGF\n2QXZ0ZSnk3h65auvZl/PP7/cOMzM2uGcc+C00+BnPyun/0ok+oGyjS/EmlmqyizfVCLRl3k12sys\nCB2f6Ht7nejNLG2XXdbhib6vzzNuzCxtS5fCli3l9F36rJvXX4e5c+HQIZg4sbBQzMwK9dZb2cek\nHjx46o9CGPezbrZuhSVLnOTNLG1Tp8LChdmz6YtWeqLfsgUuvrjsKMzM2u/ii8sp3zjRm5kVpKw6\nvRO9mVlBLr64nKdYViLRX3RR2VGYmbVfWaWbUmfd/OpXcPrpo7sKbWY23hw+nD0O4Y03YMIpDLPH\n9ayb7dthwQIneTPrDDNnZlMsX3ut2H5LTfQvvJBNrTQz6xSLFsGLLxbbZ6mJ/qWXsh/azKxTLF7c\nYYl+xw4nejPrLIsXZ4PcIpU+on/3u8uMwMysWB05oneiN7NO0lGJPgJeftmJ3sw6y6JFHVS66e/P\nPlpr1qyyIjAzK15XF7z5ZvbE3qKUluhdnzezTiQVP6ofMdFLmiqpR1KvpM2SvjhMu69K2iapT9KI\nHyPi+ryZdaoLLoCdO4vrb9JIDSLiLUnXRMQRSROBpyStjIinBtpIWg0sjogLJV0FrAVWnOy8TvRm\n1qmKTvRNlW4i4ki+OjX/nv1DmqwB7s3b9gCzJXWd7Jw7d2aPPzAz6zTz58MrrxTXX1OJXtIESb3A\nbqAeEUMftDkPGPz+tCvfN6xXX4V5J21hZpamypVuACLiGHC5pNOBRyWtiognRtNhd3c3AM88Azfc\nUANqozmNmdm4NdKIvl6vU6/XW9bfKT+mWNJfAEci4iuD9q0FHo+I7+bbW4FVEdE/5HuPP6b4vPOy\nZO9RvZl1muefhxtuyJ7g24y2P6ZY0tmSZufr04Frgb4hzdYDt+RtVgAHhib5wd5+G/buzeaTmpl1\nmoERfVEfB9JM6eY84B5JIntjuC8i/lHSbUBExLqIeFjS9ZK2A28At57shP39cPbZMKmpwpGZWVpO\nOw2mTYN9++Css9rfXzPTKzcBVzTY/40h259sttNXX4Xzz2+2tZlZerq6YM+eYhJ9KXfGOtGbWac7\n99ysulGEUhL97t0wd24ZPZuZVcO558IvflFMX6Uk+j17sg/INTPrVF1diY/onejNrNMlP6Lfu9eJ\n3sw6W/KJfs+ebHqlmVmncunGzCxxHTGid6I3s06W9PTKiKxG79KNmXWyM8+E/UMf+N4mhSf6w4dh\n4kSYMaPons3MquOMM+DgQTh2rP19FZ7oXbYxM4PJk2H69Gzw226FJ/pf/rKYZzuYmVXdGWcUU74p\nPNEfOJD9cGZmnW7OnCwntlvhif7QIZg9u+hezcyqJ9kR/aFDcPrpRfdqZlY9yY7oDx70iN7MDDyi\nNzNLnkf0ZmaJ84jezCxxyY7oPevGzCwzZ06iI/qDBz2iNzODxEs3HtGbmcGsWYk+AsEjejOzTGUS\nvaT5kh6TtFnSJkmfatBmlaQDkjbmyxeGO59H9GZmmZkz4fXX29/PpCbavA38aUT0SZoJbJD0aERs\nHdLuyYi4caSTedaNmVlm5syKjOgjYndE9OXrh4EtwLwGTdVMh4cPZ3+umJl1usok+sEkvQtYBvQ0\nOHy1pD5JD0laOtw5pk+HCaV8gKGZWbUMJPqI9vbTTOkGgLxs8wDw6XxkP9gGYEFEHJG0GngQWNLo\nPBHddHdn67VajVqtdupRm5klYNIkmDIF/u3fskHwgHq9Tr1eb1k/iibeSiRNAn4EPBIRdzbRfgdw\nZUTsG7I/Fi4MXn55lNGamSXmnHNg8+bsw8KHI4mIaKo83kizRZRvAs8Nl+QldQ1aX072BrKvUdvB\n71pmZp3utNPaX6cfsXQjaSXwUWCTpF4ggM8DC4GIiHXATZI+ARwF3gRuHu58/lBwM7MTpk/PSjft\nNGKij4ingIkjtLkLuKuZDj2iNzM7oYhEX/j8Fyd6M7MTpk2DN99sbx+FJ3qXbszMTpg2zSN6M7Ok\nuXRjZpY4l27MzBLnEb2ZWeKSHNFPm1Z0j2Zm1ZXkxdgpU4ru0cysupIs3UydWnSPZmbVlWTpxone\nzOwEj+jNzBI3daoTvZlZ0qZMgaNH29uHL8aamZVo8mT41a/a24dH9GZmJUpyRO9Eb2Z2gkf0ZmaJ\nmzIlwUTvGr2Z2Qku3ZiZJc6lGzOzxHlEb2aWOI/ozcwS54uxZmaJS7J0M3ly0T2amVVXJUo3kuZL\nekzSZkmbJH1qmHZflbRNUp+kZcOdb9KksYRrZpaWIkb0zaTdt4E/jYg+STOBDZIejYitAw0krQYW\nR8SFkq4C1gIrGp1s4sRWhG1mloZKjOgjYndE9OXrh4EtwLwhzdYA9+ZteoDZkroanc8jejOzEyp3\nMVbSu4BlQM+QQ/OAnYO2d/GbbwaAE72Z2WBVKd0AkJdtHgA+nY/sR+Wv/qr7eLKv1WrUarXRnsrM\nbNxrVLqp1+vU6/WW9aGIGLmRNAn4EfBIRNzZ4Pha4PGI+G6+vRVYFRH9Q9rF22+H6/RmZrlDh2D+\n/OzrcCQRERptH82Wbr4JPNcoyefWA7fkAa0ADgxN8sc7LHxCp5lZdRVxMXbE0o2klcBHgU2SeoEA\nPg8sBCIi1kXEw5Kul7QdeAO4dfjztSZwM7MUTJgATRRWxqSp0k3LOpOiyP7MzKru6FGYMePkF2SL\nKt2YmVkbTJgAx461uY/2nt7MzE5GcqI3M0vawHXLdla1nejNzEokZYsTvZlZwtpdp3eiNzMrWbvr\n9E70ZmYla/dceid6M7OSuXRjZpY4J3ozs8S5Rm9mljjX6M3MEufSjZlZ4pzozcwS5xq9mVniXKM3\nM0ucSzdmZolzojczS5xr9GZmiXON3swscS7dmJklzonezCxxrtGbmSWu9Bq9pLsl9Ut6dpjjqyQd\nkLQxX77Q+jDNzNLV7tLNpCbafAv4GnDvSdo8GRE3tiYkM7POUnqNPiL+Cdg/QjO1Jhwzs84zXmr0\nV0vqk/SQpKUtOqeZWUdod42+mdLNSDYACyLiiKTVwIPAkuEad3d3H1+v1WrUarUWhGBmNn4NLd3U\n63Xq9XrLzq9o4m1E0kLghxFxaRNtdwBXRsS+Bseimf7MzDrJJZfAt78Nlw6TYSUREaMukTdbuhHD\n1OEldQ1aX0725vEbSd7MzBorvXQj6X6gBpwl6efAHcAUICJiHXCTpE8AR4E3gZvbF66ZWXpKn14Z\nEf9xhON3AXe1LCIzsw5T+vRKMzNrr/EyvdLMzEap9EcgmJlZe7l0Y2aWOCd6M7PEuUZvZpY41+jN\nzBLn0o2ZWeKc6M3MEucavZlZ4lyjNzNLnEs3ZmaJc6I3M0uca/RmZolzjd7MLHEu3ZiZJc6J3sws\nca7Rm5klzjV6M7PEuXRjZpY4J3ozs8S5Rm9mljjX6M3MEld66UbS3ZL6JT17kjZflbRNUp+kZa0N\n0cwsbaUneuBbwO8Nd1DSamBxRFwI3AasbVFsZmYdofQafUT8E7D/JE3WAPfmbXuA2ZK6WhOemVn6\nxkONfh6wc9D2rnyfmZk1od2lm0ntO3Vj3d3dx9drtRq1Wq3oEMzMKuVjH4O5c09s1+t16vV6y86v\naOLvBUkLgR9GxKUNjq0FHo+I7+bbW4FVEdHfoG0005+ZmZ0giYjQaL+/2dKN8qWR9cAteTArgAON\nkryZmZVjxNKNpPuBGnCWpJ8DdwBTgIiIdRHxsKTrJW0H3gBubWfAZmZ2apoq3bSsM5duzMxOWVGl\nGzMzG6ec6M3MEudEb2aWOCd6M7PEOdGbmSXOid7MLHFO9GZmiXOiNzNLnBO9mVninOjNzBLnRG9m\nljgnejOzxDnRm5klzonezCxxTvRmZolzojczS5wTvZlZ4pzozcwS50RvZpY4J3ozs8Q50ZuZJc6J\n3swscU0lekkfkrRV0guSPtvg+CpJByRtzJcvtD5UMzMbjRETvaQJwNeB3wPeC3xE0kUNmj4ZEVfk\ny39vcZyFqtfrZYfQFMfZWuMhzvEQIzjOqmlmRL8c2BYRP4uIo8B3gDUN2qmlkZVovPzyHWdrjYc4\nx0OM4DirpplEPw/YOWj7lXzfUFdL6pP0kKSlLYnOzMzGbFKLzrMBWBARRyStBh4ElrTo3GZmNgaK\niJM3kFYA3RHxoXz7c0BExJdP8j07gCsjYt+Q/SfvzMzMGoqIUZfHmxnRPw28R9JC4DXgw8BHBjeQ\n1BUR/fn6crI3kH1DTzSWQM3MbHRGTPQR8Y6kTwKPktX0746ILZJuyw7HOuAmSZ8AjgJvAje3M2gz\nM2veiKUbMzMb3wq7M3akm66KJOluSf2Snh20b46kRyU9L+n/Spo96NifS9omaYuk6wqKcb6kxyRt\nlrRJ0qcqGudUST2SevNYv1jFOPN+J+Q39K2vaox53y9L+pf8Nf1pFWOVNFvS3+V9bpZ0VQVjXJK/\nhhvzrwclfapqcQ7qd7OkZyV9W9KUlsYZEW1fyN5QtgMLgclAH3BREX0PE8+/B5YBzw7a92Xgv+Xr\nnwW+lK8vBXrJylzvyn8OFRDjXGBZvj4TeB64qGpx5n3PyL9OBP4ZWFnROD8D/C9gfRV/54PifAmY\nM2RfpWIF/idwa74+CZhdtRiHxDsBeBW4oGpxkuXFl4Ap+fZ3gY+3Ms6iXuQVwCODtj8HfLbIX/Qw\nL+7gRL8V6MrX5wJbG8UKPAJcVUK8DwL/ocpxAjOAn+b/ECsVJzAf+AegxolEX6kYB/W3AzhryL7K\nxAqcDrzYYH9lYmwQ23XA/6tinMCcPKY5efJe3+r/60WVbpq96apM50Y+cygidgPn5vuHxr6LgmOX\n9C6yv0D+mewXX6k485JIL7AbqEfEcxWM82+B24HBF6WqFuOAAP5B0tOS/nO+r0qxvhvYK+lbeVlk\nnaQZFYtxqJuB+/P1SsUZEfuBrwA/z/s8GBE/bmWcfnrl8CpxlVrSTOAB4NMRcZjfjKv0OCPiWERc\nTjZqfr+kGhWKU9INQH9E9HHyR3WU/lrmVkbEFcD1wH+R9H4q9HqSjTqvAO7K43yDbJRZpRiPkzQZ\nuBH4u3xXpeKUtIisrLgQOB84TdJHG8Q16jiLSvS7gAWDtufn+6qkX1IXgKS5wC/y/bvI6noDCotd\n0iSyJH9fRPygqnEOiIhDwMPA+yoW50rgRkkvAf8b+ICk+4DdFYrxuIh4Lf+6h6xkt5xqvZ6vADsj\n4pl8+3tkib9KMQ62GtgQEXvz7arF+T7gqYjYFxHvAN8H/l0r4ywq0R+/6UrSFLKbrtYX1PdwxK+P\n7tYD/ylf/zjwg0H7P5xfBX838B6yOnQRvgk8FxF3VjVOSWcPzAaQNB24luxCUWXijIjPR8SCiFhE\n9m/vsYj4GPDDqsQ4QNKM/K84JJ1GVlveRLVez35gp6SBx5x8ENhcpRiH+AjZG/yAqsX5PLBC0jRJ\nIns9n2tpnAVeDPlQ/gNtAz5XVL/DxHI/2RX4t8jqYreSXQj5cR7jo8AZg9r/OdmV7S3AdQXFuBJ4\nh2yGUi+wMX8Nz6xYnJfksfUC/wL813x/peIc1PcqTlyMrVyMZPXvgd/5poH/K1WLFbiMbADXB/wf\nslk3lYox73cGsAeYNWhfFeO8nezN8lngHrLZiS2L0zdMmZklzhdjzcwS50RvZpY4J3ozs8Q50ZuZ\nJc6J3swscU70ZmaJc6I3M0ucE72ZWeL+P2CzFQA8OddaAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11437f630>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(lda_01.eig_vals.cumsum()/lda_01.eig_vals.sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1168f6eb8>]"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEACAYAAAC9Gb03AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAF11JREFUeJzt3X2QXFWdxvHvE5IJiZCACQbJCxEEY9SAREMwsmkXQQNV\n4R+q8G1Vyi2yrFtSuLq6iJWprZXdrVrLlzUlpnzFXXctXxaDSBEVWxA1SpKRbEIQSJQQSQgJ4S1R\nA/ntH/cm0ww9Mz0z3ffeOfN8qrpy+/bpe37MhKdPzrn3tiICMzNL17iyCzAzs85y0JuZJc5Bb2aW\nOAe9mVniHPRmZolz0JuZJa7loJc0TtIGSWuavLZU0v789Q2SrmtvmWZmNlzjh9D2amALMKWf1++I\niOUjL8nMzNqppRG9pFnAxcAXB2rWlorMzKytWp26+RTwYWCgy2jPk9Qj6RZJ80dempmZtcOgQS/p\nEmB3RPSQjdqbjdzXA3Mi4mzgc8BNba3SzMyGTYPd60bS9cC7gGeBScDxwHcj4t0DvGc7sDAi9vXZ\n7xvrmJkNQ0QMe3p80BF9RFwbEXMi4jTgbcDtfUNe0oyG7UVkHyD7aCIiKv9YuXJl6TW4Ttc5Wmt0\nne1/jNRQzrp5HkkrstyO1cBlkq4CDgEHgctHXJmZmbXFkII+In4K/DTf/kLD/lXAqvaWZmZm7eAr\nY5uo1Wpll9AS19leo6HO0VAjuM6qGXQxtq2dSVFkf2ZmKZBEdHIx1szMRjcHvZlZ4hz0ZmaJc9Cb\nmSXOQW9mljgHvZlZ4hz0ZmaJc9CbmSXOQW9mljgHvZlZ4hz0ZmaJc9CbmSXOQW9mlrhhf/FIVRw8\nCHv2wBNPwHPPZY/Dh8E3yRwbNMD9/KryWlXqGOi1qtQx0GtVqePYY2HaNBg/itJzFJUKu3bB978P\n69fDhg2wZQscOgQnnQQnnADHHJM9JBjnf6skb6AP86q8VpU6BnqtKnUM9FpV6gA4cAD274cTT4QF\nC2DxYnjTm6BWy/KnikbF/ej37oVrroGbb4Zly7If7DnnwKtfDVOnDvypbGbWbs89B48+mg04162D\nW27Jnl93HVx5ZfszaaT3o6980O/YARddBBdeCJ/4BBx/fIeKMzMbgQ0bYMUKWLoU/v3f23vswoJe\n0jjgbuDhiFje5PXPAsuAZ4D3RkRPkzZDDvpLLoGFC+Gf/mlIbzMzK9zevfDKV8JPfgKvelX7jlvk\nN0xdDWzpp4hlwOkRcQawArhhuAU1+tnPYPNm+NjH2nE0M7POmjYNPvQhuP76sit5vpaCXtIs4GLg\ni/00uRS4ESAi1gFTJc0YSWERWcB3d8PEiSM5kplZca68Em69FR5+uOxKerU6ov8U8GGgv3mXmcCO\nhuc7833D9vOfwyOPwLveNZKjmJkV64QTYPlyWLOm7Ep6DRr0ki4Bdudz7sofHffVr8L73je6zlU1\nMwN4y1vgttvKrqJXKzG6BFgu6WJgEnC8pBsj4t0NbXYCsxuez8r3vUB3d/fR7VqtRq1We0GbP/8Z\nvvMd2LSpherMzCrmzW+Gv/mb7DqfCROG/v56vU69Xm9bPUM6vVLSUuDv+551k38IvD8iLpG0GPh0\nRCxu8v6Wzrq5887svPm77265NDOzSlm4ED79aTj//JEfq8izbvp2vELSlQAR8QNgu6QHgC8Afzvc\n4wL86EdwwQUjOYKZWbkuugjWri27ikwlL5h64xth5crsIikzs9HottvgX/4F2jEDk9yVsU89Baec\nArt3w+TJBRVmZtZmTz6ZZdnevSM/Rby0qZtOueMOeN3rHPJmNrpNmQLz5lVjrbFyQf/jH3t+3szS\ncP752eC1bJUL+jvvzG73aWY22v3FX2SZVrZKzdEfOADTp2dzWpMmFVaWmVlH7NkDZ5yRZdpI7lWf\n1Bz93XfDa17jkDezNJx0Erz0peVf/FmpoP/FL+C888quwsysfc49F371q3JrqFTQ//znDnozS8vr\nX++gPyoiG9G/4Q1lV2Jm1j6LFsGvf11uDZUJ+m3boKsLZs8evK2Z2WixYAE88AA880x5NVQm6H/x\ni+xLv83MUjJxYva1ghs3lldDZYJ+w4bsilgzs9SUPX1TqaA/55yyqzAza7+yF2QrEfSHD2f/rHHQ\nm1mKXv96j+jZti37nsXp08uuxMys/c48M/sO7KeeKqf/SgS9p23MLGXjx8P8+fB//1dO/w56M7MC\nLFgAv/lNOX076M3MCnDWWWM46CMc9GaWvrPOgnvuKafvQYNe0kRJ6yRtlLRZ0vVN2iyVtF/Shvxx\nXasFPPIIjBuX3eHNzCxVCxZkd7E8fLj4vscP1iAi/iTpTRFxQNIxwF2SlkTEXX2a3hERy4dawObN\n2VVjZmYpO/HE7OzC7dvh9NOL7bulqZuIOJBvTszf83iTZsO6Kf6WLdlqtJlZ6sqap28p6CWNk7QR\n2AXUI2JLk2bnSeqRdIuklqPbI3ozGyvKmqcfdOoGICIOA6+VNAVYK2lpRPy0ocl6YE4+vbMMuAk4\ns9mxuru7j27XajW2bKnxjncMt3wzs9Fj/nxYs2bwdvV6nXq93rZ+h/ydsZI+DhyIiE8O0GY7sDAi\n9vXZ/7zvjI2AF78Yfvvb7Cu3zMxStn49vO990NMztPd1/DtjJU2XNDXfngRcCPT0aTOjYXsR2QfI\n80K+mV27sivGHPJmNha84hXZwLboM29ambp5KfA1SSL7YPh6RPxY0gogImI1cJmkq4BDwEHg8lY6\n90KsmY0lxx2XzWI89BDMnVtcv62cXrkJeMHlTBHxhYbtVcCqoXb+4INwxhlDfZeZ2eg1bx7cd1+x\nQV/qlbHbtsFpp5VZgZlZsV7xCti6tdg+HfRmZgWaN89Bb2aWNI/ozcwSd/rpWfYVqbSgf/xxePZZ\nmDatrArMzIo3Z052M8dDh4rrs7Sg3749G81r2JcAmJmNPhMmZHfr3bGjuD5LC/oHH/S0jZmNTXPn\nwu9+V1x/pQW95+fNbKx62cuyWY2ilBb0v/99NldlZjbWzJ07RoJ+506YNaus3s3MyjN3bjbYLUqp\nQT9zZlm9m5mVZ+bMLAOL4qA3MyvYzJnwhz8U19+Q70c/os7y+9EfOgSTJ8PBg9ltis3MxpL9+2H2\nbHjqqdbad/x+9J2wa1d2D3qHvJmNRVOnZvekbzXoR6qUoPe0jZmNZVKx8/SlBb3PuDGzseyUU4qb\npy9t6ubkk8vo2cysGpIf0e/Z4++JNbOxLfkRvYPezMa6k06Cxx4rpq9Bg17SREnrJG2UtFnS9f20\n+6yk+yX1SDp7oGPu2QPTpw+3ZDOz0W/69CwLi9DKl4P/SdKbIuKApGOAuyQtiYi7jrSRtAw4PSLO\nkHQucAOwuL9jPvaYR/RmNrZVakQPEBEH8s2J+Xse79PkUuDGvO06YKqkGf0dz1M3ZjbWFTmibyno\nJY2TtBHYBdQjYkufJjOBxtvo78z3NeWgN7Ox7qSTKjR1AxARh4HXSpoCrJW0NCJ+OpwOV67s5tFH\n4fOfhwsuqFGr1YZzGDOzUW2gqZt6vU69Xm9bX0O+142kjwMHIuKTDftuAH4SEd/Mn28FlkbE7j7v\njb17g9NOy+71YGY2VkXAxInZbRAmThy4bcfvdSNpuqSp+fYk4EKgp0+zNcC78zaLgf19Q/4IT9uY\nmWW3QZg+vZgF2Vambl4KfE2SyD4Yvh4RP5a0AoiIWB0RP5B0saQHgGeAK/o72N69MG1aW2o3MxvV\njizIdvreX62cXrkJOKfJ/i/0ef53rXT4xBPZndvMzMa6ohZkC78y9sknHfRmZgAnnJANfjut8KD3\niN7MLDNlSjb47bRSgn7KlKJ7NTOrnmSD3lM3ZmaZZIPeUzdmZplkg/7JJz11Y2YGCQe9R/RmZpmp\nUx30ZmZJS3ZE78VYM7NMskHv0yvNzDLJBr1H9GZmmWSD3nP0ZmaZZIP+8OHB771sZjYWHH98FvRD\n/FqQISs86I87LrsPs5nZWDdhAnR1wcGDne2n8KCfPLnoHs3MqquI6RsHvZlZiV70ouzrBDup8KCf\nNKnoHs3MqmvSJPjjHzvbh0f0ZmYlmjTJc/RmZkk79tgKjOglzZJ0u6TNkjZJ+kCTNksl7Ze0IX9c\n19/xPHVjZtariBH9oF8ODjwLfDAieiQdB6yXtDYitvZpd0dELB/sYB7Rm5n1qsSIPiJ2RURPvv00\ncC8ws0nTls6Od9CbmfWq3By9pLnA2cC6Ji+fJ6lH0i2S5vd3DE/dmJn1qsrUDQD5tM23gavzkX2j\n9cCciDggaRlwE3Bms+P09HTT3Z1t12o1arXa0Ks2M0tEs6mber1OvV5vWx+KFm6yIGk88H3g1oj4\nTAvttwMLI2Jfn/1x7bXBJz4x3HLNzNJyzTUwezZ88IP9t5FERAz75jGtTt18GdjSX8hLmtGwvYjs\nA2Rfs7a+oZmZWa8iLpgadOpG0hLgncAmSRuBAK4FTgUiIlYDl0m6CjgEHAQu7+94Dnozs17HHluB\nOfqIuAs4ZpA2q4BVrXTY1dVaYWZmY8GkSQne1MwjejOzXhMmwKFDne3DQW9mVqKuLge9mVnSJkyA\nP/+5s3046M3MSpTkiN6LsWZmvTyiNzNLnBdjzcwS19XlEb2ZWdI8ojczS1ySi7EOejOzXkkuxvqs\nGzOzXp66MTNLXJKLsRMmFN2jmVl1JTmiH9/yd1qZmaUvycVYB72ZWa8kF2Md9GZmvZIc0R8z4FeY\nmJmNLR7Rm5klzouxZmaJq8TUjaRZkm6XtFnSJkkf6KfdZyXdL6lH0tn9dlj4R4uZWXUVMXXTyvj6\nWeCDEdEj6ThgvaS1EbH1SANJy4DTI+IMSecCNwCLmx1MakfZZmZpGDcOIjrcx2ANImJXRPTk208D\n9wIz+zS7FLgxb7MOmCppRptrNTNLzrhxcPhwh/sYSmNJc4GzgXV9XpoJ7Gh4vpMXfhiYmVkfRQR9\ny0uj+bTNt4Gr85H9sHR3dx/drtVq1Gq14R7KzGzUaxb09Xqder3etj4ULUwOSRoPfB+4NSI+0+T1\nG4CfRMQ38+dbgaURsbtPu2ilPzOzsUTKwr6/NUxJRMSwVzhbnbr5MrClWcjn1gDvzgtaDOzvG/Jm\nZtac1NkF2UFH9JKWAHcAm4DIH9cCpwIREavzdp8D3go8A1wRERuaHMsjejOzPsaPhz/+sf/rjEY6\nom9p6qZdHPRmZi/U1QVPP93/FzMVNXVjZmYd0ukzbxz0ZmYlO7IY2ykOejOzknX66lgHvZlZyTx1\nY2aWOAe9mVniHPRmZonzYqyZWeK8GGtmljhP3ZiZJc5Bb2aWOAe9mVnivBhrZpY4L8aamSXOUzdm\nZolz0JuZJc5Bb2aWOC/GmpklzouxZmaJK33qRtKXJO2WdE8/ry+VtF/ShvxxXfvLNDNLV6eDvp/v\nHH+erwD/Adw4QJs7ImJ5e0oyMxtbSh/RR8TPgMcHaTbsbyc3MxvrRsti7HmSeiTdIml+m45pZjYm\nVGHqZjDrgTkRcUDSMuAm4Mz+Gnd3dx/drtVq1Gq1NpRgZjZ69T3rpl6vU6/X23Z8RQvn9Eg6Fbg5\nIha00HY7sDAi9jV5LVrpz8xsLFm4EFavzv5sRhIRMewp8lanbkQ/8/CSZjRsLyL78HhByJuZWXOl\nT91I+gZQA6ZJeghYCXQBERGrgcskXQUcAg4Cl3euXDOz9JQe9BHxjkFeXwWsaltFZmZjzGg568bM\nzIbJt0AwM0tc6RdMmZlZZznozcwS56A3M0ucF2PNzBLnxVgzs8R56sbMLHEOejOzxDnozcwS58VY\nM7PEeTHWzCxxnroxM0ucg97MLHEOejOzxHkx1swscV6MNTNLnKduzMwSV3rQS/qSpN2S7hmgzWcl\n3S+pR9LZ7S3RzCxtpQc98BXgLf29KGkZcHpEnAGsAG5oU21mZmNC6YuxEfEz4PEBmlwK3Ji3XQdM\nlTSjPeWZmaVvNCzGzgR2NDzfme8zM7MWVGHqxszMOqjTQT++DcfYCcxueD4r39dUd3f30e1arUat\nVmtDCWZmo9d73gMzGia86/U69Xq9bcdXtDAxJGkucHNEvKbJaxcD74+ISyQtBj4dEYv7OU600p+Z\nmfWSRERouO8fdEQv6RtADZgm6SFgJdAFRESsjogfSLpY0gPAM8AVwy3GzMzar6URfds684jezGzI\nRjqi92KsmVniHPRmZolz0JuZJc5Bb2aWOAe9mVniHPRmZolz0JuZJc5Bb2aWOAe9mVniHPRmZolz\n0JuZJc5Bb2aWOAe9mVniHPRmZolz0JuZJc5Bb2aWOAe9mVniHPRmZolz0JuZJa6loJf0VklbJf1W\n0keavL5U0n5JG/LHde0v1czMhmPQoJc0Dvgc8BbgVcDbJc1r0vSOiDgnf/xzm+ssVL1eL7uElrjO\n9hoNdY6GGsF1Vk0rI/pFwP0R8fuIOAT8D3Bpk3bD/obyqhktv3zX2V6joc7RUCO4zqppJehnAjsa\nnj+c7+vrPEk9km6RNL8t1ZmZ2YiNb9Nx1gNzIuKApGXATcCZbTq2mZmNgCJi4AbSYqA7It6aP/8o\nEBHxbwO8ZzuwMCL29dk/cGdmZtZURAx7eryVEf2vgZdLOhV4BHgb8PbGBpJmRMTufHsR2QfIvr4H\nGkmhZmY2PIMGfUQ8J+nvgLVkc/pfioh7Ja3IXo7VwGWSrgIOAQeByztZtJmZtW7QqRszMxvdCrsy\ndrCLrook6UuSdku6p2HfiZLWSrpP0m2Spja89o+S7pd0r6SLCqpxlqTbJW2WtEnSBypa50RJ6yRt\nzGu9vop15v2Oyy/oW1PVGvO+fyfpN/nP9FdVrFXSVEnfyvvcLOncCtZ4Zv4z3JD/+YSkD1StzoZ+\nN0u6R9J/Sepqa50R0fEH2QfKA8CpwASgB5hXRN/91PNG4GzgnoZ9/wb8Q779EeBf8+35wEayaa65\n+X+HCqjxZODsfPs44D5gXtXqzPuenP95DPBLYElF67wG+E9gTRV/5w11bgNO7LOvUrUCXwWuyLfH\nA1OrVmOfescBfwBmV61OslzcBnTlz78JvKeddRb1Q14M3Nrw/KPAR4r8Rffzw20M+q3AjHz7ZGBr\ns1qBW4FzS6j3JuDNVa4TmAz8Kv+LWKk6gVnAD4EavUFfqRob+tsOTOuzrzK1AlOAB5vsr0yNTWq7\nCLizinUCJ+Y1nZiH95p2/79e1NRNqxddleklkZ85FBG7gJfk+/vWvpOCa5c0l+xfIL8k+8VXqs58\nSmQjsAuoR8SWCtb5KeDDQOOiVNVqPCKAH0r6taS/zvdVqdaXAY9J+ko+LbJa0uSK1djX5cA38u1K\n1RkRjwOfBB7K+3wiIn7Uzjp998r+VWKVWtJxwLeBqyPiaV5YV+l1RsThiHgt2aj5fEk1KlSnpEuA\n3RHRw8C36ij9Z5lbEhHnABcD75d0PhX6eZKNOs8BVuV1PkM2yqxSjUdJmgAsB76V76pUnZJOI5tW\nPBU4BXiRpHc2qWvYdRYV9DuBOQ3PZ+X7qmS3pBkAkk4GHs337ySb1zuisNoljScL+a9HxPeqWucR\nEfEk8APgdRWrcwmwXNI24L+Bv5T0dWBXhWo8KiIeyf/cQzZlt4hq/TwfBnZExN358++QBX+Vamy0\nDFgfEY/lz6tW5+uAuyJiX0Q8B/wv8IZ21llU0B+96EpSF9lFV2sK6rs/4vmjuzXAe/Pt9wDfa9j/\ntnwV/GXAy8nmoYvwZWBLRHymqnVKmn7kbABJk4ALyRaKKlNnRFwbEXMi4jSyv3u3R8RfATdXpcYj\nJE3O/xWHpBeRzS1volo/z93ADklHbnNyAbC5SjX28XayD/gjqlbnfcBiScdKEtnPc0tb6yxwMeSt\n+X/Q/cBHi+q3n1q+QbYC/yeyebEryBZCfpTXuBY4oaH9P5KtbN8LXFRQjUuA58jOUNoIbMh/hi+u\nWJ2vyWvbCPwG+FC+v1J1NvS9lN7F2MrVSDb/feR3vunI/ytVqxU4i2wA1wN8l+ysm0rVmPc7GdgD\nHN+wr4p1fpjsw/Ie4GtkZye2rU5fMGVmljgvxpqZJc5Bb2aWOAe9mVniHPRmZolz0JuZJc5Bb2aW\nOAe9mVniHPRmZon7f03q0Sa2qGoCAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11680a588>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(lda_35.eig_vals.cumsum()/lda_35.eig_vals.sum())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Display Eigen Vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import scipy as sp\n",
    "from scipy.misc import toimage, imresize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAMgAAADICAAAAACIM/FCAAAGRklEQVR4nO2X227jVhJF9y6SLbXt\nnplg/v8h3xcgg87EgWSSteehDinq6u7EmBQatQDLEnlE1qrLoc2fAXRd11nX+TxN8zSx7/qu7zDP\n0zxP6ruu7zqLc/PQ930/9OM4jm/jaEPfD32PaZ6neUbf9X3fc5qmeZq8a9cRAMHd3X2mkUaTXHIJ\nKwJgRjNa33d91/dvx+PxeDzSzDoznpae3nYvLy8vX16eARh+EEokGz0BEJDkLkXXSpJTAkiScRrx\nCZDcXQJpFp8RK0FCcgqkdWZGQg4A0HoFLDdYBuSy+SU5ZwCzu0CjxW1v4tN4HGwVAQQ55BGTcPIw\nkYAgCAQNFOSzC6CZkZQECKCBaFo0QGa2WELteCQNi04Ef4qSgCiP60yzwCaC2yby+a3r5NuKRKKx\nVMQJCaDJ2uVPFXFzSbHJAIBHRWypCKJjyaiIllTdrQih9oMwAdRpml2wDrR7HoBPo9GntSKE0FpL\n7S5OAcRSVgkkadFacIGMXLVcs6VWToAwWy67JE9r1FGfdk1gGyYFSg5J9GitFsJNFflk0PS2qQjk\nPG3scjAKb4pg47h5nFXcQ4sIAJhAQm4t8fQW/rYip3g2FVknIFIGp8yN7q1hNyuuKwKfxw6bGdnW\nu1WErSSAYhIIkhKcLoAwklQkeJOQhSjvEjhOM8LVYr3DmYkoOCSJMXl3e8uheTQ7icQ4u7fJlTwi\nI+MibWOCAZBIb90UM9MCWqKIh7fNclfbtc4ixdK+avfYRrkIt/ePHw+afdkt+ov7kACvKql3Pl+e\nlrjuuHch7rbMO9c/W9nWnolQpMVUc7O0PWhwtlfG75uRiJKDbTO/DQERD7r/e7moiNreuVxesfVg\no34ezU0EN6dv9tgbEFo8vr0A99mKtCRjyZTWx1d7poVn/Kw7/5WNAMDJRxpcr0VC/ACV84qAoK4K\nLvB2k9xtC0V/4dGQnLLyIVwO++lniYjA0lxhuSw9fy5sUZO/2Y/t68L5GP5Vzod96ZmLqM5W3Hx/\n+Y3olkcBflgpGv3VkY+5w0ck+bv4Yf4fKZFslEg2SiQbJZKNEslGiWSjRLJRItkokWyUSDZKJBsl\nko0SyUaJZKNEslEi2SiRbJRINkokGyWSjRLJRolko0SyUSLZKJFslEg2SiQbJZKNEslGiWSjRLJR\nItkokWyUSDZKJBslko0SyUaJZKNEslEi2SiRbJRINkokGyWSjRLJRolko0SyUSLZKJFslEg2SiQb\nJZKNEslGiWSjRLJRItkokWyUSDZKJBslko0SyUaJZKNEslEi2SiRbJRINkokGyWSjRLJRolko0Sy\nUSLZKJFslEg2SiQbJZKNEslGiWSjRLJRItkokWz0f3cAAACXJImkkbyxgIAeX+IDRYRbIdyH7VuA\nT9M4jaMP/dD3w2VIjKWEHtl8lIjaCzcHHnu1sxSg8Xg4HI7jfr/f7e0qJILvafzNrcUWmqbD6+vr\n6/Hl6cVtuLFuabcHJh8j0urB9WU9eq8qp34hpPHw+tvXr8fDP8Rhv4mYABjAAYX55uRHi/wZGBWh\nAE3H19/+8+sfo1u/n69XkgZYrL3HB86IWnjrgXdmn2Csg6bD69dff/ndOezfput1iIqI/4dh/0to\nno6HP/77+9Pz8W2+jjVaiw8Lci7CzZ74zra9/c66VtujFNUKw7Y5xdRwvYna2m7Yvxze5qd///Tl\nadebzu8vQQLiUaM1yIsYtyIEQX67B09vYtp5GvAWvQAqtk9EPNEom1gJ2qf98+h8/ulfX553PSFK\nFyJcTCi2zXjNxJUIWgm/ox4XtHmMzab1f3wyOZZPDBNJIAHChs9vju7wzy9fnnaDeTwlYy9As4iL\nCeIpdfcqsnTjcs9HcPtGIKTIsgDKTlkTNwmSIJK0dS0I0Ib9zH53fHl6ftr1ZIt5yY50qggYV9Bl\nvi9mhDRC8D9ZEgEQFWn2CLaZxJREPs3klOQwEGT3ydHtPo+f9/vdriepjUhTiJelwLiQvZoRGg3O\nGNT34Pqy/oWybLsESVGCIBK06BRBUXYBkIOUgRzQ7aZx+jQMQ9+b2tKWnGaPFrkiK9R5sq9mxAwQ\nv8XjBlHtpZu4rcgmzaSJrRlFkBy6Ty73zszMLiqy7Fbt1/K0v6zID/P/SIlk43/0fnMKqNodnwAA\nAABJRU5ErkJggg==\n",
      "text/plain": [
       "<PIL.Image.Image image mode=L size=200x200 at 0x116922A20>"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "toimage(imresize(lda_01.eig_vecs[0].reshape(28, 28), size=(200, 200)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAMgAAADICAAAAACIM/FCAAAE0klEQVR4nO2W3W7bOBCFzyEpW45T\nZLHv/xr7TL3oAg3Q7NaJyTl7Qco/iVvA/fN0MV8uEllSOJ8OZ2T+BYAgQQIggNZqrbUi55xzpjVr\n1pRSTiknM2tmljoyMzMhMSUmSCaT2JEgQQRJUOg/A4IAAeH42Qm88NnXSVff4ZQQ8Ub5DCCnlBNz\nq9VqrUYBQiKBsa1JkHy1cSUJAAicnOK3bPAfQPkMoJQCMre6e97tdii5lFxIcvFQnwVLhSQgjbPU\nMiUAcPEePXy4Scv5nyoyrZCQMtrun6enT5zn9TyXlABA7JXxUiLsxcpNIoZUxMy6e/r4+Ji291uU\ndRqRQCT6DAUkgBQIiJIk9uoP6fXJyv4E+pWE0J/H6fgFDtf/EJF/ASiVBibU3afHvz/kP6rKPCpH\n74OkxNcrjkBehUDodomwTAZm1N3Txw/vp6o8V6TxvlpKJc97hLjgQWg5Hj2iX9kjebU2pYy6e3r8\n8H6lPN/XMZiPc+l1jyy7pLf6ico5vyqeUgDkTMiaMU/r7btpu1lN+WIBh+Jvsnu+StkAWJdMtaq0\n2j481/Lnw/08vX1TCmDvdoz2fnP+hpQZwKokqO0tre8eGvPDw/3mrUjv7TFwL1nwpiplBjBNmWp7\nS6u7ipK399t5OpmPAACxt7dEqL8djzI6DIabuZQZQCmLyJbTJs3zfGlrQaDO39eOKDOAlBPVZGnF\nsnnHqUzl0tY6n1TnLj62FkGaTGk9bUxgevt9BHj1Vn7LjZs9L39JIH/fb/W/b+WvCBFvhIg3QsQb\nIeKNEPFGiHgjRLwRIt4IEW+EiDdCxBsh4o0Q8UaIeCNEvBEi3ggRb4SIN0LEGyHijRDxRoh4I0S8\nESLeCBFvhIg3QsQbIeKNEPFGiHgjRLwRIt4IEW+EiDdCxBsh4o0Q8UaIeCNEvBEi3ggRb4SIN0LE\nGyHijRDxRoh4I0S8ESLeCBFvhIg3QsQbIeKNEPFGiHgjRLwRIt4IEW+EiDdCxBsh4o0Q8UaIeCNE\nvBEi3ggRb4SIN0LEGyHijRDxRoh4I0S8ESLeCBFvhIg3QsQbIeKNEPFGiHgjRLwRIt4IEW+EiDfK\nrQv4EgJ4zfWRyM9Ex19fSYVnF0ciP5tre6ScB/QDywAIggAEXV3X1ZQJgCRI3+5yuJMEAZPJDDnn\nknNu1lqzxg4ECSJOJKFFuR8Cy1mAgJYFeHpInD+aMmGsfG0oPCgIhwhAEjJrrWmVmFerab+vMKtM\nKYGjEC1xiT2uYbL8JxJcRERpKZ1YbHmUHyIFgBoNdp3HOYIgEmCiIKu1WppSWc/r57xrsJoEkqkn\noH7tUQsEyXGS44GgF3ww715jwUMiQ6YnwgaRX4zk9MR5pGNtQEOl12pt/6LJWFabu8RWYTUVpv5U\nJQkjBaEfUQQJ0ST1NQ6JHCvg+GBccra3ygSgAdL3JSJI/QGRUKv7F1uLZb3ZyvbPUM3smTQAMI5a\nKQBSVk+kHxJLIovJYVgcvQ6yOhEhlHTtUDnpkV6NJJApibJaX1o15tXd1tpLplUxS0gix2zpg4H9\nRi57axwSTEvNVM9A/QaN9XieSikAIDP7rumo7gEct1arYlnN2/1zSbCKbBpTDUv39kTGrX2oAcv/\nOdSjMVa/kMjgf/NmDxFv/AcItB8Hsr6YaQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<PIL.Image.Image image mode=L size=200x200 at 0x116A45E80>"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "toimage(imresize(lda_35.eig_vecs[0].reshape(28, 28), size=(200, 200)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}