analisis respuestas.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn import preprocessing\n",
    "from sklearn import tree\n",
    "from sklearn.model_selection import train_test_split\n",
    "import graphviz \n",
    "import warnings\n",
    "import numpy as np\n",
    "import sqlite3\n",
    "import json\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.metrics import roc_auc_score, roc_curve, log_loss, confusion_matrix, precision_score, recall_score, classification_report, accuracy_score\n",
    "import joblib\n",
    "from joblib import dump, load\n",
    "from pathlib import Path\n",
    "import datetime\n",
    "\n",
    "%matplotlib inline\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('csvs/data.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cand_data = json.loads(open('predictor_pol/candidatos.json','r').read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_party(id):\n",
    "    for p in cand_data:\n",
    "        for c in p['candidates']:\n",
    "            if c['id'] == id:\n",
    "                return p['party']\n",
    "    return 'n/a'\n",
    "\n",
    "def get_name(id):\n",
    "    for p in cand_data:\n",
    "        for c in p['candidates']:\n",
    "            if c['id'] == id:\n",
    "                return c['name']\n",
    "    return 'n/a'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['partido'] = df.candidatoId.apply(get_party)\n",
    "df['nombre'] = df.candidatoId.apply(get_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Análisis exploratorio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"cantidad de registros: {}\".format(len(df)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.rcParams['figure.figsize'] = [10, 5]\n",
    "df.groupby('partido').size().sort_values(ascending=False).plot.bar(title='Partido elegido por usuarios')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.groupby('nombre').size().sort_values(ascending=False).head(10).plot.bar(title='Candidato elegido por usuarios')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "by_day = pd.read_csv('csvs/encuestas_por_dia.csv')\n",
    "by_day = by_day.sort_values(by='dia',ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "by_day.plot.barh(x='dia',y='cant',title='Cantidad de encuestas por dia')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = datetime.datetime.now() - datetime.timedelta(days=6)\n",
    "by_day = by_day[by_day['dia'] > d.strftime(\"%Y-%m-%d\")]\n",
    "by_day.plot.barh(x='dia',y='cant',title='Por dia (ultimos 5 dias)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "by_candidate = df[['nombre','fecha']]\n",
    "by_candidate[\"dia\"] = by_candidate.fecha.apply(lambda x: x[0:10])\n",
    "by_candidate.drop(columns=['fecha'],inplace=True)\n",
    "by_candidate.head(10)\n",
    "#TODO aca me gustaria hacer unas graficas por dia por candidato"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Entrenamiento de modelo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = ['1', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19',\n",
    "       '2', '20', '21', '22', '23', '24', '25', '26', '3', '4', '5', '6', '7',\n",
    "       '8', '9']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_total(row, features):\n",
    "    total = 0\n",
    "    for i in features:\n",
    "        total += row[i]\n",
    "    return total\n",
    "\n",
    "df['suma_respuestas'] = df.apply(lambda row: add_total(row, features),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "todo_5 = len(df[df['suma_respuestas'] == 26*5])\n",
    "todo_3 = len(df[df['suma_respuestas'] == 26*3])\n",
    "todo_1 = len(df[df['suma_respuestas'] == 26])\n",
    "print('cant encuestas todo 5: {}'.format(todo_5))\n",
    "print('cant encuestas todo 1: {}'.format(todo_1))\n",
    "print('cant encuestas todo 3: {}'.format(todo_3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#TODO conviene sacar las todo 5 y todo 1 ?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LogisticRegression (en produccion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#Split train and test\n",
    "df_train, df_test = train_test_split(df, test_size=0.20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Try to predict candidate\n",
    "clf = LogisticRegression(random_state=0, solver='lbfgs',multi_class='multinomial',max_iter=700).fit(df_train[features], df_train.candidatoId)\n",
    "df_test['predicted_candidate'] = clf.predict(df_test[features])\n",
    "\n",
    "df_test['predicted_candidate'] = df_test['predicted_candidate'].apply(get_name)\n",
    "df_test['candidato'] = df_test['candidatoId'].apply(get_name)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(classification_report(df_test.nombre,df_test.predicted_candidate))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### este pareceria ser el mejor de los 3, se pone en produccion"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# El modelo original"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_original = df[df.id < 206]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(df_original)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n=17\n",
    "k=7\n",
    "\n",
    "#entreno con el df original y testeo con el resto de los datos\n",
    "\n",
    "pca = PCA(n_components=n)\n",
    "pca.fit(df_original[features])\n",
    "x_train = pca.transform(df_original[features])\n",
    "y_train = df_original.candidatoId\n",
    "x_test = pca.transform(df[df.id>=206][features])\n",
    "y_test = df[df.id>=206].candidatoId\n",
    "\n",
    "candidate_model = KNeighborsClassifier(n_neighbors=n)\n",
    "candidate_model.fit(x_train, y_train)\n",
    "results = pd.DataFrame()\n",
    "results['truth'] = y_test\n",
    "results['truth'] = results.truth.apply(get_name)\n",
    "results['prediction'] = candidate_model.predict(x_test)\n",
    "results['prediction'] = results.prediction.apply(get_name)\n",
    "\n",
    "print(classification_report(results.truth,results.prediction ))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ahora probamos un nuevo KNeighborsClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n=22\n",
    "k=22\n",
    "\n",
    "pca = PCA(n_components=n)\n",
    "pca.fit(df[features])\n",
    "X = pca.transform(df[features])\n",
    "y = df.candidatoId\n",
    "\n",
    "#entreno con el 80% de los datos\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20)\n",
    "candidate_model = KNeighborsClassifier(n_neighbors=n)\n",
    "candidate_model.fit(X_train, y_train)\n",
    "results = pd.DataFrame()\n",
    "results['truth'] = y_test\n",
    "results['truth'] = results.truth.apply(get_name)\n",
    "results['prediction'] = candidate_model.predict(X_test)\n",
    "results['prediction'] = results.prediction.apply(get_name)\n",
    "\n",
    "print(classification_report(results.truth,results.prediction ))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gana el LR"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}