Exercise+9.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualization Callback Example\n",
    "\n",
    "\n",
    "## Preamble\n",
    "Before we dive into creating a callback, we'll need a simple model to work with.  This tutorial uses a model similar to the one in neon's `examples/mnist_mlp.py`, but the same callback should apply to any model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from neon.backends import gen_backend\n",
    "from neon.data import MNIST\n",
    "from neon.data import ArrayIterator\n",
    "from neon.initializers import Gaussian\n",
    "from neon.layers import Affine\n",
    "from neon.transforms import Rectlin, Softmax\n",
    "from neon.models import Model\n",
    "from neon.layers import GeneralizedCost\n",
    "from neon.transforms import CrossEntropyMulti\n",
    "from neon.optimizers import GradientDescentMomentum\n",
    "\n",
    "be = gen_backend(batch_size=128)\n",
    "\n",
    "mnist = MNIST(path='data/')\n",
    "train_set = mnist.train_iter\n",
    "test_set = mnist.valid_iter\n",
    "\n",
    "init_norm = Gaussian(loc=0.0, scale=0.01)\n",
    "\n",
    "layers = []\n",
    "layers.append(Affine(nout=100, init=init_norm, activation=Rectlin()))\n",
    "layers.append(Affine(nout=10, init=init_norm,\n",
    "                     activation=Softmax()))\n",
    "\n",
    "mlp = Model(layers=layers)\n",
    "cost = GeneralizedCost(costfunc=CrossEntropyMulti())\n",
    "optimizer = GradientDescentMomentum(0.1, momentum_coef=0.9)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Callbacks\n",
    "Neon provides an API for calling operations during the model fit. The progress bars displayed during training are an example of a callback, and we'll go through the process of adding a new callback that visualizes cost graphically instead of printing to screen.\n",
    "\n",
    "To make a new callback, subclass from `Callback`, and implement the desired callback methods.  \n",
    "\n",
    "Each of the callback functions have access to `callback_data` and `model` objects.  `callback_data` is an H5 file that is saved when supplying the `-o` flag to neon, and callbacks should store any computed data into `callback_data`.  Visualization callbacks can read already computed data such as training or validation cost from `callback_data`.\n",
    "\n",
    "This callback implements the subset of the available callback functions that it needs:\n",
    "http://neon.nervanasys.com/docs/latest/callbacks.html#creating-callbacks\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from neon.callbacks.callbacks import Callbacks, Callback\n",
    "from bokeh.plotting import output_notebook, figure, ColumnDataSource, show\n",
    "from bokeh.io import push_notebook\n",
    "from timeit import default_timer\n",
    "\n",
    "class CostVisCallback(Callback):\n",
    "    \"\"\"\n",
    "    Callback providing a live updating console based progress bar.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, epoch_freq=1,\n",
    "                 minibatch_freq=1, update_thresh_s=0.65):\n",
    "        super(CostVisCallback, self).__init__(epoch_freq=epoch_freq,\n",
    "                                                  minibatch_freq=minibatch_freq)\n",
    "        self.update_thresh_s = update_thresh_s\n",
    "        \n",
    "        output_notebook()\n",
    "        \n",
    "        self.fig = figure(name=\"cost\", title=\"Cost\", x_axis_label=\"Epoch\", plot_width=900)\n",
    "        self.train_source = ColumnDataSource(data=dict(x=[], y0=[]))\n",
    "        self.train_cost = self.fig.line(x=[], y=[], source=self.train_source)\n",
    "        \n",
    "        self.val_source = ColumnDataSource(data=dict(x=[], y0=[]))\n",
    "        self.val_cost = self.fig.line(x=[], y=[], source=self.val_source, color='red')\n",
    "        \n",
    "\n",
    "    def on_train_begin(self, callback_data, model, epochs):\n",
    "        \"\"\"\n",
    "        A good place for one-time startup operations, such as displaying the figure.\n",
    "        \"\"\"\n",
    "        show(self.fig)\n",
    "\n",
    "    def on_epoch_begin(self, callback_data, model, epoch):\n",
    "        \"\"\"\n",
    "        Since the number of minibatches per epoch is not constant, calculate it here.\n",
    "        \"\"\"\n",
    "        self.start_epoch = self.last_update = default_timer()\n",
    "        self.nbatches = model.nbatches\n",
    "\n",
    "    def on_minibatch_end(self, callback_data, model, epoch, minibatch):\n",
    "        \"\"\"\n",
    "        Read the training cost already computed by the TrainCostCallback out of 'callback_data', and display it.\n",
    "        \"\"\"\n",
    "        now = default_timer()\n",
    "        mb_complete = minibatch + 1\n",
    "        \n",
    "        mbstart = callback_data['time_markers/minibatch'][epoch-1] if epoch > 0 else 0\n",
    "        train_cost = callback_data['cost/train'][mbstart + minibatch]\n",
    "\n",
    "        mb_epoch_scale = epoch + minibatch / float(self.nbatches)\n",
    "        self.train_source.data['x'].append(mb_epoch_scale)\n",
    "        self.train_source.data['y'].append(train_cost)\n",
    "            \n",
    "        if (now - self.last_update > self.update_thresh_s or mb_complete == self.nbatches):\n",
    "            self.last_update = now\n",
    "\n",
    "            push_notebook()\n",
    "\n",
    "    def on_epoch_end(self, callback_data, model, epoch):\n",
    "        \"\"\"\n",
    "        If per-epoch validation cost is being computed by the LossCallback, plot that too. \n",
    "        \"\"\"\n",
    "        _eil = self._get_cached_epoch_loss(callback_data, model, epoch, 'loss')\n",
    "        if _eil:\n",
    "            self.val_source.data['x'].append(1 + epoch)\n",
    "            self.val_source.data['y'].append(_eil['cost'])\n",
    "            push_notebook()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running our callback\n",
    "\n",
    "We'll create all of the standard neon callbacks, and then add ours at the end.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "callbacks = Callbacks(mlp, eval_set=test_set)\n",
    "cv = CostVisCallback()\n",
    "callbacks.add_callback(cv)\n",
    "mlp.fit(train_set, optimizer=optimizer, num_epochs=10, cost=cost, callbacks=callbacks)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}