Timed Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import datetime\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "#%matplotlib tk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "legco_df = pd.read_csv('records-all-with-info.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>161700.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>80849.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>46678.913601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>40424.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>80849.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>121274.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>161699.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Unnamed: 0\n",
       "count  161700.000000\n",
       "mean    80849.500000\n",
       "std     46678.913601\n",
       "min         0.000000\n",
       "25%     40424.750000\n",
       "50%     80849.500000\n",
       "75%    121274.250000\n",
       "max    161699.000000"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "legco_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>topic_id</th>\n",
       "      <th>member_id</th>\n",
       "      <th>vote</th>\n",
       "      <th>vote-date</th>\n",
       "      <th>vote-time</th>\n",
       "      <th>motion-en</th>\n",
       "      <th>mover-en</th>\n",
       "      <th>mover-type</th>\n",
       "      <th>vote-separate-mechanism</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>17/10/2012-1</td>\n",
       "      <td>曾鈺成</td>\n",
       "      <td>Present</td>\n",
       "      <td>17/10/2012</td>\n",
       "      <td>19:37:53</td>\n",
       "      <td>AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...</td>\n",
       "      <td>Dr Kenneth CHAN</td>\n",
       "      <td>Member</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>17/10/2012-1</td>\n",
       "      <td>何俊仁</td>\n",
       "      <td>Yes</td>\n",
       "      <td>17/10/2012</td>\n",
       "      <td>19:37:53</td>\n",
       "      <td>AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...</td>\n",
       "      <td>Dr Kenneth CHAN</td>\n",
       "      <td>Member</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>17/10/2012-1</td>\n",
       "      <td>李卓人</td>\n",
       "      <td>Yes</td>\n",
       "      <td>17/10/2012</td>\n",
       "      <td>19:37:53</td>\n",
       "      <td>AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...</td>\n",
       "      <td>Dr Kenneth CHAN</td>\n",
       "      <td>Member</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>17/10/2012-1</td>\n",
       "      <td>涂謹申</td>\n",
       "      <td>Yes</td>\n",
       "      <td>17/10/2012</td>\n",
       "      <td>19:37:53</td>\n",
       "      <td>AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...</td>\n",
       "      <td>Dr Kenneth CHAN</td>\n",
       "      <td>Member</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>17/10/2012-1</td>\n",
       "      <td>陳鑑林</td>\n",
       "      <td>No</td>\n",
       "      <td>17/10/2012</td>\n",
       "      <td>19:37:53</td>\n",
       "      <td>AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...</td>\n",
       "      <td>Dr Kenneth CHAN</td>\n",
       "      <td>Member</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0      topic_id member_id     vote   vote-date vote-time  \\\n",
       "0           0  17/10/2012-1       曾鈺成  Present  17/10/2012  19:37:53   \n",
       "1           1  17/10/2012-1       何俊仁      Yes  17/10/2012  19:37:53   \n",
       "2           2  17/10/2012-1       李卓人      Yes  17/10/2012  19:37:53   \n",
       "3           3  17/10/2012-1       涂謹申      Yes  17/10/2012  19:37:53   \n",
       "4           4  17/10/2012-1       陳鑑林       No  17/10/2012  19:37:53   \n",
       "\n",
       "                                           motion-en         mover-en  \\\n",
       "0  AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...  Dr Kenneth CHAN   \n",
       "1  AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...  Dr Kenneth CHAN   \n",
       "2  AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...  Dr Kenneth CHAN   \n",
       "3  AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...  Dr Kenneth CHAN   \n",
       "4  AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...  Dr Kenneth CHAN   \n",
       "\n",
       "  mover-type vote-separate-mechanism  \n",
       "0     Member                     Yes  \n",
       "1     Member                     Yes  \n",
       "2     Member                     Yes  \n",
       "3     Member                     Yes  \n",
       "4     Member                     Yes  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "legco_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>topic_id</th>\n",
       "      <th>member_id</th>\n",
       "      <th>vote</th>\n",
       "      <th>vote-date</th>\n",
       "      <th>vote-time</th>\n",
       "      <th>motion-en</th>\n",
       "      <th>mover-en</th>\n",
       "      <th>mover-type</th>\n",
       "      <th>vote-separate-mechanism</th>\n",
       "      <th>dt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>17/10/2012-1</td>\n",
       "      <td>曾鈺成</td>\n",
       "      <td>Present</td>\n",
       "      <td>17/10/2012</td>\n",
       "      <td>19:37:53</td>\n",
       "      <td>AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...</td>\n",
       "      <td>Dr Kenneth CHAN</td>\n",
       "      <td>Member</td>\n",
       "      <td>Yes</td>\n",
       "      <td>2012-10-17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>17/10/2012-1</td>\n",
       "      <td>何俊仁</td>\n",
       "      <td>Yes</td>\n",
       "      <td>17/10/2012</td>\n",
       "      <td>19:37:53</td>\n",
       "      <td>AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...</td>\n",
       "      <td>Dr Kenneth CHAN</td>\n",
       "      <td>Member</td>\n",
       "      <td>Yes</td>\n",
       "      <td>2012-10-17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>17/10/2012-1</td>\n",
       "      <td>李卓人</td>\n",
       "      <td>Yes</td>\n",
       "      <td>17/10/2012</td>\n",
       "      <td>19:37:53</td>\n",
       "      <td>AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...</td>\n",
       "      <td>Dr Kenneth CHAN</td>\n",
       "      <td>Member</td>\n",
       "      <td>Yes</td>\n",
       "      <td>2012-10-17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>17/10/2012-1</td>\n",
       "      <td>涂謹申</td>\n",
       "      <td>Yes</td>\n",
       "      <td>17/10/2012</td>\n",
       "      <td>19:37:53</td>\n",
       "      <td>AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...</td>\n",
       "      <td>Dr Kenneth CHAN</td>\n",
       "      <td>Member</td>\n",
       "      <td>Yes</td>\n",
       "      <td>2012-10-17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>17/10/2012-1</td>\n",
       "      <td>陳鑑林</td>\n",
       "      <td>No</td>\n",
       "      <td>17/10/2012</td>\n",
       "      <td>19:37:53</td>\n",
       "      <td>AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...</td>\n",
       "      <td>Dr Kenneth CHAN</td>\n",
       "      <td>Member</td>\n",
       "      <td>Yes</td>\n",
       "      <td>2012-10-17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0      topic_id member_id     vote   vote-date vote-time  \\\n",
       "0           0  17/10/2012-1       曾鈺成  Present  17/10/2012  19:37:53   \n",
       "1           1  17/10/2012-1       何俊仁      Yes  17/10/2012  19:37:53   \n",
       "2           2  17/10/2012-1       李卓人      Yes  17/10/2012  19:37:53   \n",
       "3           3  17/10/2012-1       涂謹申      Yes  17/10/2012  19:37:53   \n",
       "4           4  17/10/2012-1       陳鑑林       No  17/10/2012  19:37:53   \n",
       "\n",
       "                                           motion-en         mover-en  \\\n",
       "0  AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...  Dr Kenneth CHAN   \n",
       "1  AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...  Dr Kenneth CHAN   \n",
       "2  AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...  Dr Kenneth CHAN   \n",
       "3  AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...  Dr Kenneth CHAN   \n",
       "4  AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...  Dr Kenneth CHAN   \n",
       "\n",
       "  mover-type vote-separate-mechanism         dt  \n",
       "0     Member                     Yes 2012-10-17  \n",
       "1     Member                     Yes 2012-10-17  \n",
       "2     Member                     Yes 2012-10-17  \n",
       "3     Member                     Yes 2012-10-17  \n",
       "4     Member                     Yes 2012-10-17  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def from_str_to_date(date_str):\n",
    "    t = date_str.split('/')\n",
    "    day = int(t[0])\n",
    "    month = int(t[1])\n",
    "    year = int(t[2])\n",
    "    dt = datetime.datetime(year, month, day)\n",
    "    return dt\n",
    "\n",
    "legco_df['dt'] = legco_df['vote-date'].apply(from_str_to_date)\n",
    "legco_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "71"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(legco_df.member_id.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "70"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def clean_record(t):\n",
    "    if t == 'Dr Joseph LEE':\n",
    "        t = 'Prof Joseph LEE'\n",
    "    # https://github.com/code4hk/legcohk/issues/1\n",
    "    if t == '郭偉强':\n",
    "        t = '郭偉強'\n",
    "    # Other normalization if any\n",
    "    # ...\n",
    "    return t\n",
    "legco_df.member_id = legco_df.member_id.apply(clean_record)\n",
    "len(legco_df.member_id.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def to_numeric(x):\n",
    "    if x == 'Yes':\n",
    "        return 1\n",
    "    elif x == 'No':\n",
    "        return -1\n",
    "    else:\n",
    "        return 0\n",
    "legco_df['vote'] = legco_df['vote'].apply(to_numeric)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>topic_id</th>\n",
       "      <th>member_id</th>\n",
       "      <th>vote</th>\n",
       "      <th>vote-date</th>\n",
       "      <th>vote-time</th>\n",
       "      <th>motion-en</th>\n",
       "      <th>mover-en</th>\n",
       "      <th>mover-type</th>\n",
       "      <th>vote-separate-mechanism</th>\n",
       "      <th>dt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>17/10/2012-1</td>\n",
       "      <td>曾鈺成</td>\n",
       "      <td>0</td>\n",
       "      <td>17/10/2012</td>\n",
       "      <td>19:37:53</td>\n",
       "      <td>AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...</td>\n",
       "      <td>Dr Kenneth CHAN</td>\n",
       "      <td>Member</td>\n",
       "      <td>Yes</td>\n",
       "      <td>2012-10-17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>17/10/2012-1</td>\n",
       "      <td>何俊仁</td>\n",
       "      <td>1</td>\n",
       "      <td>17/10/2012</td>\n",
       "      <td>19:37:53</td>\n",
       "      <td>AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...</td>\n",
       "      <td>Dr Kenneth CHAN</td>\n",
       "      <td>Member</td>\n",
       "      <td>Yes</td>\n",
       "      <td>2012-10-17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>17/10/2012-1</td>\n",
       "      <td>李卓人</td>\n",
       "      <td>1</td>\n",
       "      <td>17/10/2012</td>\n",
       "      <td>19:37:53</td>\n",
       "      <td>AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...</td>\n",
       "      <td>Dr Kenneth CHAN</td>\n",
       "      <td>Member</td>\n",
       "      <td>Yes</td>\n",
       "      <td>2012-10-17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>17/10/2012-1</td>\n",
       "      <td>涂謹申</td>\n",
       "      <td>1</td>\n",
       "      <td>17/10/2012</td>\n",
       "      <td>19:37:53</td>\n",
       "      <td>AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...</td>\n",
       "      <td>Dr Kenneth CHAN</td>\n",
       "      <td>Member</td>\n",
       "      <td>Yes</td>\n",
       "      <td>2012-10-17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>17/10/2012-1</td>\n",
       "      <td>陳鑑林</td>\n",
       "      <td>-1</td>\n",
       "      <td>17/10/2012</td>\n",
       "      <td>19:37:53</td>\n",
       "      <td>AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...</td>\n",
       "      <td>Dr Kenneth CHAN</td>\n",
       "      <td>Member</td>\n",
       "      <td>Yes</td>\n",
       "      <td>2012-10-17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0      topic_id member_id  vote   vote-date vote-time  \\\n",
       "0           0  17/10/2012-1       曾鈺成     0  17/10/2012  19:37:53   \n",
       "1           1  17/10/2012-1       何俊仁     1  17/10/2012  19:37:53   \n",
       "2           2  17/10/2012-1       李卓人     1  17/10/2012  19:37:53   \n",
       "3           3  17/10/2012-1       涂謹申     1  17/10/2012  19:37:53   \n",
       "4           4  17/10/2012-1       陳鑑林    -1  17/10/2012  19:37:53   \n",
       "\n",
       "                                           motion-en         mover-en  \\\n",
       "0  AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...  Dr Kenneth CHAN   \n",
       "1  AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...  Dr Kenneth CHAN   \n",
       "2  AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...  Dr Kenneth CHAN   \n",
       "3  AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...  Dr Kenneth CHAN   \n",
       "4  AMENDMENT BY DR HON KENNETH CHAN TO HON IP KIN...  Dr Kenneth CHAN   \n",
       "\n",
       "  mover-type vote-separate-mechanism         dt  \n",
       "0     Member                     Yes 2012-10-17  \n",
       "1     Member                     Yes 2012-10-17  \n",
       "2     Member                     Yes 2012-10-17  \n",
       "3     Member                     Yes 2012-10-17  \n",
       "4     Member                     Yes 2012-10-17  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "legco_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>topic_id</th>\n",
       "      <th>member_id</th>\n",
       "      <th>vote</th>\n",
       "      <th>dt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17/10/2012-1</td>\n",
       "      <td>曾鈺成</td>\n",
       "      <td>0</td>\n",
       "      <td>2012-10-17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>17/10/2012-1</td>\n",
       "      <td>何俊仁</td>\n",
       "      <td>1</td>\n",
       "      <td>2012-10-17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>17/10/2012-1</td>\n",
       "      <td>李卓人</td>\n",
       "      <td>1</td>\n",
       "      <td>2012-10-17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>17/10/2012-1</td>\n",
       "      <td>涂謹申</td>\n",
       "      <td>1</td>\n",
       "      <td>2012-10-17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>17/10/2012-1</td>\n",
       "      <td>陳鑑林</td>\n",
       "      <td>-1</td>\n",
       "      <td>2012-10-17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       topic_id member_id  vote         dt\n",
       "0  17/10/2012-1       曾鈺成     0 2012-10-17\n",
       "1  17/10/2012-1       何俊仁     1 2012-10-17\n",
       "2  17/10/2012-1       李卓人     1 2012-10-17\n",
       "3  17/10/2012-1       涂謹申     1 2012-10-17\n",
       "4  17/10/2012-1       陳鑑林    -1 2012-10-17"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "legco_df_selected = legco_df[['topic_id', 'member_id', 'vote', 'dt']]\n",
    "legco_df_selected.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_pc1_from_df(df):\n",
    "    df_matrix = pd.DataFrame(index=df['member_id'].unique())\n",
    "    for gn, g in df.groupby('topic_id'):\n",
    "        df_matrix[gn] = g.set_index('member_id')['vote']\n",
    "    \n",
    "    #NOTE: \n",
    "    #    For more rigorous study, \n",
    "    #    you need to investigate what are those missing values\n",
    "    df_matrix.fillna(0)\n",
    "    def clean_values(x):\n",
    "        x[(x != 1) & (x != -1)] = 0\n",
    "    df_matrix.apply(clean_values)\n",
    "    \n",
    "    X = np.matrix(df_matrix.as_matrix()).astype('float')\n",
    "    X = X - np.mean(X, 0)\n",
    " \n",
    "    from sklearn.decomposition import PCA\n",
    "    pca = PCA(n_components=1)\n",
    "    # X is of dimensions: n_samples x n_features\n",
    "    X_reduced = pca.fit_transform(X)\n",
    "    df_pc1 = pd.DataFrame(X_reduced, index=df_matrix.index, columns=['PC1'])\n",
    "    df_pc1 = df_pc1.sort('PC1')\n",
    "    #plt.figure(figsize=(12, 20))\n",
    "    #plt.plot(df_pc1['PC1'], df_pc1.rank()['PC1'], 'd', markersize=10)\n",
    "    #for (_x, _y, _s) in zip(df_pc1['PC1'], df_pc1.rank()['PC1'], df_pc1.index):\n",
    "    #    plt.annotate(_s.decode('utf-8'), (_x, _y), xytext=(_x + 0.01, _y - 0.02))\n",
    "    #plt.title('Spectrum from Principal Component 1')\n",
    "    #plt.show()\n",
    "    if df_pc1.T['梁國雄'][0] > df_pc1.T['曾鈺成'][0]:\n",
    "        # Fix the relative order of landmark nodes\n",
    "        df_pc1['PC1'] = - df_pc1['PC1'] \n",
    "    return df_pc1\n",
    "\n",
    "df_pc1 = get_pc1_from_df(legco_df_selected)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2015-02-13 00:00:00\n",
      "2012-10-17 00:00:00\n"
     ]
    }
   ],
   "source": [
    "print legco_df_selected.dt.max()\n",
    "print legco_df_selected.dt.min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Elapse: 44.2592389584\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/site-packages/matplotlib/pyplot.py:424: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).\n",
      "  max_open_warning, RuntimeWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "2013-01-26    21\n",
       "2013-02-26    24\n",
       "2013-03-26    24\n",
       "2013-04-26    24\n",
       "2013-05-26     1\n",
       "2013-06-26     1\n",
       "2013-07-26     1\n",
       "2013-08-26     1\n",
       "2013-09-26     1\n",
       "2013-10-26     1\n",
       "2013-11-26     1\n",
       "2013-12-26     1\n",
       "2014-01-26     1\n",
       "2014-02-26     1\n",
       "2014-03-26     1\n",
       "2014-04-26     1\n",
       "2014-05-26     4\n",
       "2014-06-26     1\n",
       "2014-07-26     1\n",
       "2014-08-26     1\n",
       "2014-09-26     1\n",
       "2014-10-26     1\n",
       "2014-11-26     1\n",
       "2014-12-26     1\n",
       "2015-01-26     1\n",
       "2015-02-26     1\n",
       "Name: 梁國雄, dtype: float64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import time\n",
    "#from multiprocessing import Pool \n",
    "# I have 4 cores. Change this to fit you\n",
    "#p = Pool(4)\n",
    "\n",
    "from dateutil.relativedelta import relativedelta \n",
    "\n",
    "init_date = datetime.datetime(2012, 12, 26)\n",
    "cutoff_dates = []\n",
    "cur_date = init_date\n",
    "while cur_date < legco_df_selected.dt.max():\n",
    "    legco_df_selected_cut = legco_df_selected[legco_df_selected.dt < cur_date]\n",
    "    cur_date += relativedelta(months=1)\n",
    "    cutoff_dates.append(cur_date)\n",
    "cutoff_dates.append(cur_date)\n",
    "\n",
    "def get_1year_records_till(t_end):\n",
    "    t_begin = t_end - relativedelta(years=1)\n",
    "    return legco_df_selected[(legco_df_selected.dt >= t_begin) & (legco_df_selected.dt < t_end)]\n",
    "\n",
    "def get_pc1_at_cutoff_date(t):\n",
    "    return (t, get_pc1_from_df(get_1year_records_till(t)))\n",
    "\n",
    "_begin_time = time.time()\n",
    "# This runs for: 47.6116089821 \n",
    "PCs = map(lambda t: (t, get_pc1_from_df(get_1year_records_till(t))), cutoff_dates)\n",
    "# This runs for\n",
    "#PCs = p.map(get_pc1_at_cutoff_date, cutoff_dates)\n",
    "_end_time = time.time()\n",
    "print 'Elapse:', _end_time - _begin_time\n",
    "\n",
    "scores = pd.DataFrame(index=legco_df_selected.member_id.unique())\n",
    "for (t, PC) in PCs:\n",
    "    scores[t] = PC\n",
    "\n",
    "rankings = scores.rank()\n",
    "rankings.T['梁國雄']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>曾鈺成</th>\n",
       "      <th>何俊仁</th>\n",
       "      <th>李卓人</th>\n",
       "      <th>涂謹申</th>\n",
       "      <th>陳鑑林</th>\n",
       "      <th>梁耀忠</th>\n",
       "      <th>劉皇發</th>\n",
       "      <th>劉慧卿</th>\n",
       "      <th>譚耀宗</th>\n",
       "      <th>石禮謙</th>\n",
       "      <th>...</th>\n",
       "      <th>葉建源</th>\n",
       "      <th>葛珮帆</th>\n",
       "      <th>廖長江</th>\n",
       "      <th>潘兆平</th>\n",
       "      <th>鄧家彪</th>\n",
       "      <th>蔣麗芸</th>\n",
       "      <th>盧偉國</th>\n",
       "      <th>鍾國斌</th>\n",
       "      <th>鍾樹根</th>\n",
       "      <th>謝偉銓</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>26.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>26.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>29.153846</td>\n",
       "      <td>16.538462</td>\n",
       "      <td>11.538462</td>\n",
       "      <td>18.769231</td>\n",
       "      <td>58.576923</td>\n",
       "      <td>16.423077</td>\n",
       "      <td>44.076923</td>\n",
       "      <td>13.500000</td>\n",
       "      <td>65.461538</td>\n",
       "      <td>38.769231</td>\n",
       "      <td>...</td>\n",
       "      <td>16.538462</td>\n",
       "      <td>54.307692</td>\n",
       "      <td>48.692308</td>\n",
       "      <td>54.192308</td>\n",
       "      <td>49.038462</td>\n",
       "      <td>46.076923</td>\n",
       "      <td>67.538462</td>\n",
       "      <td>46.384615</td>\n",
       "      <td>43.730769</td>\n",
       "      <td>58.076923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>3.002563</td>\n",
       "      <td>3.127053</td>\n",
       "      <td>2.420426</td>\n",
       "      <td>3.723522</td>\n",
       "      <td>7.642895</td>\n",
       "      <td>7.782920</td>\n",
       "      <td>6.632786</td>\n",
       "      <td>5.368426</td>\n",
       "      <td>6.034771</td>\n",
       "      <td>10.726818</td>\n",
       "      <td>...</td>\n",
       "      <td>2.564851</td>\n",
       "      <td>7.604048</td>\n",
       "      <td>4.937767</td>\n",
       "      <td>11.668828</td>\n",
       "      <td>8.567290</td>\n",
       "      <td>3.772777</td>\n",
       "      <td>1.333590</td>\n",
       "      <td>8.542023</td>\n",
       "      <td>10.286137</td>\n",
       "      <td>2.261381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>26.000000</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>42.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>47.000000</td>\n",
       "      <td>31.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>38.000000</td>\n",
       "      <td>46.000000</td>\n",
       "      <td>31.000000</td>\n",
       "      <td>29.000000</td>\n",
       "      <td>40.000000</td>\n",
       "      <td>65.000000</td>\n",
       "      <td>40.000000</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>55.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>28.000000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>16.250000</td>\n",
       "      <td>58.000000</td>\n",
       "      <td>6.250000</td>\n",
       "      <td>39.250000</td>\n",
       "      <td>10.250000</td>\n",
       "      <td>66.000000</td>\n",
       "      <td>31.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>52.250000</td>\n",
       "      <td>46.000000</td>\n",
       "      <td>52.000000</td>\n",
       "      <td>51.000000</td>\n",
       "      <td>42.000000</td>\n",
       "      <td>67.000000</td>\n",
       "      <td>40.250000</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>56.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>28.000000</td>\n",
       "      <td>16.500000</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>59.500000</td>\n",
       "      <td>19.500000</td>\n",
       "      <td>40.000000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>68.000000</td>\n",
       "      <td>35.500000</td>\n",
       "      <td>...</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>54.500000</td>\n",
       "      <td>47.000000</td>\n",
       "      <td>54.000000</td>\n",
       "      <td>52.000000</td>\n",
       "      <td>47.500000</td>\n",
       "      <td>68.000000</td>\n",
       "      <td>44.000000</td>\n",
       "      <td>37.000000</td>\n",
       "      <td>59.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>29.000000</td>\n",
       "      <td>19.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>22.750000</td>\n",
       "      <td>63.000000</td>\n",
       "      <td>22.750000</td>\n",
       "      <td>53.000000</td>\n",
       "      <td>16.750000</td>\n",
       "      <td>69.000000</td>\n",
       "      <td>38.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>19.000000</td>\n",
       "      <td>62.000000</td>\n",
       "      <td>47.000000</td>\n",
       "      <td>66.000000</td>\n",
       "      <td>53.750000</td>\n",
       "      <td>48.000000</td>\n",
       "      <td>68.000000</td>\n",
       "      <td>46.000000</td>\n",
       "      <td>55.000000</td>\n",
       "      <td>60.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>36.000000</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>23.000000</td>\n",
       "      <td>67.000000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>53.000000</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>70.000000</td>\n",
       "      <td>63.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>62.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>68.000000</td>\n",
       "      <td>56.000000</td>\n",
       "      <td>53.000000</td>\n",
       "      <td>70.000000</td>\n",
       "      <td>66.000000</td>\n",
       "      <td>59.000000</td>\n",
       "      <td>62.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 70 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             曾鈺成        何俊仁        李卓人        涂謹申        陳鑑林        梁耀忠  \\\n",
       "count  26.000000  26.000000  26.000000  26.000000  26.000000  26.000000   \n",
       "mean   29.153846  16.538462  11.538462  18.769231  58.576923  16.423077   \n",
       "std     3.002563   3.127053   2.420426   3.723522   7.642895   7.782920   \n",
       "min    26.000000  12.000000   8.000000  12.000000  42.000000   6.000000   \n",
       "25%    28.000000  14.000000  10.000000  16.250000  58.000000   6.250000   \n",
       "50%    28.000000  16.500000  12.000000  20.000000  59.500000  19.500000   \n",
       "75%    29.000000  19.000000  13.000000  22.750000  63.000000  22.750000   \n",
       "max    36.000000  22.000000  16.000000  23.000000  67.000000  25.000000   \n",
       "\n",
       "             劉皇發        劉慧卿        譚耀宗        石禮謙    ...            葉建源  \\\n",
       "count  26.000000  26.000000  26.000000  26.000000    ...      26.000000   \n",
       "mean   44.076923  13.500000  65.461538  38.769231    ...      16.538462   \n",
       "std     6.632786   5.368426   6.034771  10.726818    ...       2.564851   \n",
       "min    36.000000   4.000000  47.000000  31.000000    ...      13.000000   \n",
       "25%    39.250000  10.250000  66.000000  31.000000    ...      15.000000   \n",
       "50%    40.000000  14.000000  68.000000  35.500000    ...      16.000000   \n",
       "75%    53.000000  16.750000  69.000000  38.000000    ...      19.000000   \n",
       "max    53.000000  22.000000  70.000000  63.000000    ...      21.000000   \n",
       "\n",
       "             葛珮帆        廖長江        潘兆平        鄧家彪        蔣麗芸        盧偉國  \\\n",
       "count  26.000000  26.000000  26.000000  26.000000  26.000000  26.000000   \n",
       "mean   54.307692  48.692308  54.192308  49.038462  46.076923  67.538462   \n",
       "std     7.604048   4.937767  11.668828   8.567290   3.772777   1.333590   \n",
       "min    38.000000  46.000000  31.000000  29.000000  40.000000  65.000000   \n",
       "25%    52.250000  46.000000  52.000000  51.000000  42.000000  67.000000   \n",
       "50%    54.500000  47.000000  54.000000  52.000000  47.500000  68.000000   \n",
       "75%    62.000000  47.000000  66.000000  53.750000  48.000000  68.000000   \n",
       "max    62.000000  60.000000  68.000000  56.000000  53.000000  70.000000   \n",
       "\n",
       "             鍾國斌        鍾樹根        謝偉銓  \n",
       "count  26.000000  26.000000  26.000000  \n",
       "mean   46.384615  43.730769  58.076923  \n",
       "std     8.542023  10.286137   2.261381  \n",
       "min    40.000000  35.000000  55.000000  \n",
       "25%    40.250000  35.000000  56.000000  \n",
       "50%    44.000000  37.000000  59.000000  \n",
       "75%    46.000000  55.000000  60.000000  \n",
       "max    66.000000  59.000000  62.000000  \n",
       "\n",
       "[8 rows x 70 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rankings.T.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "rankings.T[['梁國雄', '曾鈺成']].plot()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "rankings.T.plot()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "rankings_var = pd.DataFrame(rankings.T.var(), columns=['var'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['\\xe5\\xa7\\x9a\\xe6\\x80\\x9d\\xe6\\xa6\\xae', '\\xe6\\xa2\\x81\\xe7\\xbe\\x8e\\xe8\\x8a\\xac', '\\xe7\\x94\\xb0\\xe5\\x8c\\x97\\xe8\\xbe\\xb0', '\\xe9\\x8d\\xbe\\xe6\\xa8\\xb9\\xe6\\xa0\\xb9', '\\xe9\\xbb\\x83\\xe6\\xaf\\x93\\xe6\\xb0\\x91', '\\xe7\\x9f\\xb3\\xe7\\xa6\\xae\\xe8\\xac\\x99', '\\xe9\\x99\\xb3\\xe5\\x81\\xa5\\xe6\\xb3\\xa2', '\\xe9\\xba\\xa5\\xe7\\xbe\\x8e\\xe5\\xa8\\x9f', '\\xe6\\xbd\\x98\\xe5\\x85\\x86\\xe5\\xb9\\xb3']\n"
     ]
    }
   ],
   "source": [
    "var_kings = list(rankings_var.sort('var')[-10:-1].index)\n",
    "print var_kings\n",
    "rankings.T[var_kings].plot()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "rankings.to_csv('rankings.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "len(legco_df_selected['member_id'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "legco_df_selected.index = legco_df_selected['dt'].values\n",
    "vote_num = legco_df_selected.resample('1w', how='count')\n",
    "vote_num[['vote']].plot()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.664852255054\n",
      "0.753575357536\n"
     ]
    }
   ],
   "source": [
    "# The volumne at the peak week\n",
    "vote_num.sort('dt', ascending=False)\n",
    "print 1.0 * vote_num.ix['2014-06-01']['dt'] / vote_num.dt[\n",
    "    (vote_num.index < datetime.datetime(2014,8,1))\n",
    "    &\n",
    "    (vote_num.index >= datetime.datetime(2013,8,31))\n",
    "].sum()\n",
    "print 1.0 * vote_num.ix['2013-05-19']['dt'] / vote_num.dt[\n",
    "    (vote_num.index < datetime.datetime(2013,8,1))\n",
    "    &\n",
    "    (vote_num.index >= datetime.datetime(2012,8,31))\n",
    "].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "d1 = df_pc1.rank()\n",
    "d2 = df_pc1\n",
    "d2['PC1'] = -d2['PC1']\n",
    "d2 = d2.rank()\n",
    "d1['another'] = d2['PC1']\n",
    "d1\n",
    "df_pc1.T['梁國雄']\n",
    "df_pc1.T['曾鈺成'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_matrix = pd.DataFrame(index=legco_df_selected['member_id'].unique())\n",
    "for gn, g in legco_df_selected.groupby('topic_id'):\n",
    "    df_matrix[gn] = g.set_index('member_id')['vote']\n",
    "df_matrix.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print df_matrix.shape\n",
    "print np.product(df_matrix.shape)\n",
    "print len(legco_df_selected)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_matrix.fillna(0)\n",
    "df_matrix.describe().T.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_matrix.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}