visu.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time\n",
    "import os\n",
    "import matplotlib.pyplot as plt, mpld3\n",
    "import seaborn as sns\n",
    "from vqa.lib.logger import Experiment\n",
    "import json\n",
    "import random\n",
    "#%matplotlib notebook\n",
    "#mpld3.enable_notebook()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def plot_line(data_x, data_y, xlabel=\"epochs\", ylabel=\"accuracy %\", title=\"Name\"):\n",
    "    fig, ax = plt.subplots()\n",
    "    ax.set_xlabel(\"epochs\")\n",
    "    ax.set_ylabel(\"accuracy\")\n",
    "    ax.set_title(title)\n",
    "\n",
    "    lines = ax.plot(data_y, data_x, '--', marker=\"o\", label=\"lol\")\n",
    "\n",
    "    mpld3.plugins.connect(fig, mpld3.plugins.LineLabelTooltip(lines[0], label=xp_name))\n",
    "    mpld3.plugins.connect(fig, mpld3.plugins.PointLabelTooltip(lines[0]))#data_y)\n",
    "    mpld3.plugins.connect(fig, mpld3.plugins.MousePosition(fontsize=14))\n",
    "    #ax.grid(color='lightgrey', linestyle='solid')\n",
    "    mpld3.show()\n",
    "    \n",
    "def plot_acc(train_accs, val_accs, fig=None, \\\n",
    "             xlabel=\"epochs\", ylabel=\"accuracy %\", title=\"Name\"):\n",
    "    if fig is None:\n",
    "        fig = plt.figure()\n",
    "        n = 0\n",
    "    else:\n",
    "        n = len(fig.axes)\n",
    "        for i in range(n):\n",
    "            fig.axes[i].change_geometry(n+1, 1, i+1)\n",
    "    \n",
    "    ax = fig.add_subplot(n+1, 1, n+1)\n",
    "    \n",
    "    ax.set_xlabel(xlabel)\n",
    "    ax.set_ylabel(ylabel)\n",
    "    ax.set_title(title)\n",
    "\n",
    "    data_x = range(1, len(train_accs)+1)\n",
    "    ax.plot(data_x, train_accs, '--', marker=\"o\", label=\"train\")\n",
    "    ax.plot(data_x, val_accs, '--', marker=\"o\", label=\"val\")\n",
    "\n",
    "    line_labels = ['train', 'val']\n",
    "    point_labels = [train_accs, val_accs]\n",
    "    for i, line in enumerate(ax.lines):\n",
    "        mpld3.plugins.connect(fig, \\\n",
    "            mpld3.plugins.LineLabelTooltip(line, label=line_labels[i]))\n",
    "        mpld3.plugins.connect(fig, \\\n",
    "            mpld3.plugins.PointLabelTooltip(line, labels=point_labels[i]))\n",
    "    \n",
    "    mpld3.plugins.connect(fig, mpld3.plugins.MousePosition(fontsize=14))\n",
    "    #ax.grid(color='lightgrey', linestyle='solid')\n",
    "    #mpld3.show(open_browser=False)\n",
    "    return fig\n",
    "    \n",
    "def plot_accs(train_accs=[], val_accs=[], names=[], fig=None, \\\n",
    "             xlabel=\"epochs\", ylabel=\"accuracy %\", title=\"Name\"):\n",
    "    if fig is None:\n",
    "        fig = plt.figure()\n",
    "        n = 0\n",
    "    else:\n",
    "        n = len(fig.axes)\n",
    "        for i in range(n):\n",
    "            fig.axes[i].change_geometry(n+1, 1, i+1)\n",
    "    \n",
    "    ax = fig.add_subplot(n+1, 1, n+1)\n",
    "    \n",
    "    ax.set_xlabel(xlabel)\n",
    "    ax.set_ylabel(ylabel)\n",
    "    ax.set_title(title)\n",
    "\n",
    "    max_length = max([len(acc) for acc in val_accs])\n",
    "    data_x = range(1, max_length+1)\n",
    "    for i in range(len(val_accs)):\n",
    "        color = (random.random(), random.random(), random.random())\n",
    "        data_x_train = range(1, len(train_accs[i])+1)\n",
    "        data_x_val = range(1, len(val_accs[i])+1)\n",
    "        ax.plot(data_x_train, train_accs[i],\n",
    "                '--', marker=\"o\", color=color,\n",
    "                label=names[i]+\"train\")\n",
    "        ax.plot(data_x_val, val_accs[i],\n",
    "                '-', marker=\"o\", color=color,\n",
    "                label=names[i]+\"val\")\n",
    "\n",
    "    point_labels = {\n",
    "        'train': train_accs,\n",
    "        'val': val_accs\n",
    "    }\n",
    "    for line_id in range(len(ax.lines)):\n",
    "        label = 'train' if line_id%2==0 else 'val'\n",
    "        #mpld3.plugins.connect(fig, \\\n",
    "        #        mpld3.plugins.LineLabelTooltip(ax.lines[line_id], label=label))\n",
    "        mpld3.plugins.connect(fig, \\\n",
    "                mpld3.plugins.PointLabelTooltip(ax.lines[line_id], labels=point_labels[label][int(i/2)]))\n",
    "    \n",
    "    ax.legend()\n",
    "    labels = []\n",
    "    for name in names:\n",
    "        labels.append(name + '_train')\n",
    "        labels.append(name + '_val')\n",
    "    mpld3.plugins.connect(fig, mpld3.plugins.InteractiveLegendPlugin(ax.lines, labels))\n",
    "    mpld3.plugins.connect(fig, mpld3.plugins.MousePosition(fontsize=14))\n",
    "    #ax.grid(color='lightgrey', linestyle='solid')\n",
    "    #mpld3.show(open_browser=False)\n",
    "    return fig    \n",
    "\n",
    "def load_accs_oe(path_logger):\n",
    "    dir_xp = os.path.dirname(path_logger)\n",
    "    epochs = []\n",
    "    for name in os.listdir(dir_xp):\n",
    "        if name.startswith('epoch'):\n",
    "            epochs.append(name)\n",
    "    epochs = sorted(epochs, key=lambda x: float(x.split('_')[1]))\n",
    "    accs = {}\n",
    "    for i, epoch in enumerate(epochs):\n",
    "        epoch_id = i+1\n",
    "        path_acc = os.path.join(dir_xp, epoch, 'OpenEnded_mscoco_val2014_model_accuracy.json')\n",
    "        if os.path.exists(path_acc):\n",
    "            with open(path_acc, 'r') as f:\n",
    "                data = json.load(f)\n",
    "                accs[epoch_id] = data['overall']\n",
    "    return accs\n",
    "\n",
    "def sort(dict_):\n",
    "    return [v for k,v in sorted(dict_.items(), \\\n",
    "                           key=lambda x: float(x[0]))]\n",
    "\n",
    "def reduce(list_, num=15):\n",
    "    tmp = []\n",
    "    for i, val in enumerate(list_):\n",
    "        if i < num:\n",
    "            tmp.append(val)\n",
    "    return tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dir_logs = 'logs/vqa'\n",
    "xps = []\n",
    "for p in sorted(os.listdir(dir_logs)):\n",
    "    if not p.startswith('.'): print(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "path_logger = os.path.join(dir_logs, 'mlb_att_samplingans,True', 'logger.json')\n",
    "xp = Experiment.from_json(path_logger)\n",
    "xp.logged['val']['acc1_oe'] = load_accs_oe(path_logger)\n",
    "#print(xp.logged['val']['acc1_oe'])\n",
    "print(xp.options)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Display accuracy & loss of one exp\n",
    "def display_xp(path_logger):\n",
    "    xp = Experiment.from_json(path_logger)\n",
    "    xp.logged['val']['acc1_oe'] = load_accs_oe(path_logger)\n",
    "\n",
    "    path_visu = os.path.join(os.path.dirname(path_logger),\n",
    "                             'visu.html')\n",
    "\n",
    "    val_acc1 = sort(xp.logged['val']['acc1_oe'])\n",
    "    train_acc1 = sort(xp.logged['train']['acc1'])\n",
    "    loss_val = sort(xp.logged['val']['loss'])\n",
    "    loss_train = sort(xp.logged['train']['loss'])\n",
    "\n",
    "    max_length = min([len(val_acc1), len(train_acc1)])\n",
    "    val_acc1 = reduce(val_acc1, max_length)\n",
    "    train_acc1= reduce(train_acc1, max_length)\n",
    "    loss_val = reduce(loss_val, max_length)\n",
    "    loss_train=reduce(loss_train, max_length)\n",
    "    \n",
    "    print(max(val_acc1))\n",
    "    \n",
    "    fig = plt.figure(figsize=(10,13))\n",
    "    fig = plot_acc(train_acc1, val_acc1, fig=fig, \\\n",
    "                  title='max: '+str(max(val_acc1)))\n",
    "    fig = plot_acc(loss_train, loss_val, fig=fig, \\\n",
    "                   ylabel='loss', title='min: '+str(min(loss_val)))\n",
    "    fig.tight_layout()\n",
    "    with open(path_visu, 'w') as f:\n",
    "        mpld3.save_html(fig, f)\n",
    "        \n",
    "#for name in ['blocmutan']:#['seq2vec/bgru', 'mutan_noatt_2', 'mlb_att_2']:\n",
    "for name in os.listdir(dir_logs):\n",
    "    if not name.startswith('.'):\n",
    "        path_logger = os.path.join(dir_logs, name, 'logger.json')\n",
    "        print(name)\n",
    "        display_xp(path_logger)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Display accuracy & loss of multiple exp\n",
    "accs_train = []\n",
    "accs_val = []\n",
    "names = []\n",
    "acc_max = ('xp name', -1)\n",
    "fig = plt.figure(figsize=(20,13))\n",
    "#for name in os.listdir(dir_logs):#['bgu_normal', 'mutan_noatt', 'mutan_noatt_2', 'mlb_att_2',  'mlb_att_3']:\n",
    "#for name in ['seq2vec/bgru_dropout,25%', 'mutan_noatt', 'blocmutan_noatt', 'mlb_noatt_2lstm', 'mlb_noatt_2lstm_emb']:\n",
    "#for name in ['blocmutan_noatt', 'blocmutan_noatt_samplingans,True']:\n",
    "dir_logs = 'logs/vqa'\n",
    "exps = []\n",
    "exps.append('blocmutan_att_2')\n",
    "exps.append('mutan_att_2')\n",
    "exps.append('mlb_att_2')\n",
    "\n",
    "for name in exps:\n",
    "    if not name.startswith('.'):\n",
    "        path_logger = os.path.join(dir_logs, name, 'logger.json')\n",
    "        xp = Experiment.from_json(path_logger)\n",
    "        xp.logged['val']['acc1_oe'] = load_accs_oe(path_logger)\n",
    "        train_acc1 = sort(xp.logged['train']['acc1'])\n",
    "        val_acc1 = sort(xp.logged['val']['acc1_oe'])\n",
    "\n",
    "        max_length = min([len(val_acc1), len(train_acc1)])\n",
    "        val_acc1 = reduce(val_acc1, max_length)\n",
    "        train_acc1= reduce(train_acc1, max_length)\n",
    "\n",
    "        accs_val.append(val_acc1)\n",
    "        accs_train.append(train_acc1)\n",
    "        print(name, max(val_acc1))\n",
    "        if max(val_acc1) > acc_max[1]:\n",
    "            acc_max = (name, max(val_acc1))\n",
    "        names.append(name)\n",
    "fig = plot_accs(accs_train, accs_val, names,\n",
    "                title=acc_max[0]+' '+str(acc_max[1]), fig=fig)\n",
    "path_visu = dir_logs+'/test.html'\n",
    "with open(path_visu, 'w') as f:\n",
    "    mpld3.save_html(fig, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Display accuracy & loss of multiple exp\n",
    "accs_train = []\n",
    "accs_val = []\n",
    "names = []\n",
    "acc_max = ('xp name', -1)\n",
    "fig = plt.figure(figsize=(20,13))\n",
    "#for name in os.listdir(dir_logs):#['bgu_normal', 'mutan_noatt', 'mutan_noatt_2', 'mlb_att_2',  'mlb_att_3']:\n",
    "#for name in ['seq2vec/bgru_dropout,25%', 'mutan_noatt', 'blocmutan_noatt', 'mlb_noatt_2lstm', 'mlb_noatt_2lstm_emb']:\n",
    "#for name in ['blocmutan_noatt', 'blocmutan_noatt_samplingans,True']:\n",
    "\n",
    "dir_logs = 'logs/vqa'\n",
    "list_dir = []\n",
    "for name in os.listdir(dir_logs):\n",
    "    if not name.startswith('.') and not name.endswith('.html'):\n",
    "        list_dir.append(name)\n",
    "        \n",
    "for name in list_dir:\n",
    "    path_logger = os.path.join(dir_logs, name, 'logger.json')\n",
    "    #xp = Experiment.from_json(path_logger)\n",
    "    val_acc1 = load_accs_oe(path_logger)\n",
    "    #train_acc1 = sort(xp.logged['train']['acc1'])\n",
    "    val_acc1 = sort(val_acc1)\n",
    "\n",
    "    max_length = len(val_acc1)\n",
    "    val_acc1 = reduce(val_acc1, max_length)\n",
    "    #train_acc1= reduce(train_acc1, max_length)\n",
    "\n",
    "    accs_val.append(val_acc1)\n",
    "    #accs_train.append(train_acc1)\n",
    "    print(name, max(val_acc1))\n",
    "    if max(val_acc1) > acc_max[1]:\n",
    "        acc_max = (name, max(val_acc1))\n",
    "    names.append(name)\n",
    "\n",
    "fig = plot_accs(accs_val, accs_val, names,\n",
    "                title=acc_max[0]+' '+str(acc_max[1]), fig=fig)\n",
    "path_visu = dir_logs+'/test.html'\n",
    "with open(path_visu, 'w') as f:\n",
    "    mpld3.save_html(fig, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Test displaying accuracy of one exp\n",
    "val_acc1 = sort(xp.logged['val']['acc1'])\n",
    "train_acc1 = sort(xp.logged['train']['acc1'])\n",
    "loss_val = sort(xp.logged['val']['loss'])\n",
    "loss_train = sort(xp.logged['train']['loss'])\n",
    "\n",
    "#plot_line(val_acc1, range(1,len(val_acc1)+1))\n",
    "for i in range(1,40):\n",
    "    val_acc1_tmp = reduce(val_acc1, i)\n",
    "    train_acc1_tmp = reduce(train_acc1, i)\n",
    "    loss_val_tmp = reduce(loss_val, i)\n",
    "    loss_train_tmp = reduce(loss_train, i)\n",
    "    fig = plt.figure(figsize=(10,13))\n",
    "    fig = plot_acc(train_acc1_tmp, val_acc1_tmp, fig=fig, \\\n",
    "                  title='max: '+str(max(val_acc1_tmp)))\n",
    "    fig = plot_acc(loss_train_tmp, loss_val_tmp, fig=fig, \\\n",
    "                   ylabel='loss', title='min: '+str(min(loss_val_tmp)))\n",
    "    fig.tight_layout()\n",
    "    with open('test.html', 'w') as f:\n",
    "        mpld3.save_html(fig, f)\n",
    "    time.sleep(3)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:vqa]",
   "language": "python",
   "name": "conda-env-vqa-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}