P03-C02-mlp-gluon.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Multilayer perceptrons in ``gluon``\n",
    "\n",
    "Using gluon, we only need two additional lines of code to transform our logisitc regression model into a multilayer perceptron."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import mxnet as mx\n",
    "from mxnet import nd, autograd\n",
    "from mxnet import gluon\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll also want to set the compute context for our modeling. Feel free to go ahead and change this to mx.gpu(0) if you're running on an appropriately endowed machine."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ctx = mx.cpu()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The MNIST dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mnist = mx.test_utils.get_mnist()\n",
    "batch_size = 64\n",
    "train_data = mx.io.NDArrayIter(mnist[\"train_data\"], mnist[\"train_label\"], batch_size, shuffle=True)\n",
    "test_data = mx.io.NDArrayIter(mnist[\"test_data\"], mnist[\"test_label\"], batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define the model\n",
    "\n",
    "*Here's the only real difference. We add two lines!*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "net = gluon.nn.Sequential()\n",
    "with net.name_scope():\n",
    "    net.add(gluon.nn.Dense(128, activation=\"relu\"))\n",
    "    net.add(gluon.nn.Dense(64, activation=\"relu\"))\n",
    "    net.add(gluon.nn.Dense(10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parameter initialization\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "net.collect_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Softmax cross-entropy loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "loss = gluon.loss.SoftmaxCrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': .1})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluate_accuracy(data_iterator, net):\n",
    "    acc = mx.metric.Accuracy()\n",
    "    data_iterator.reset()\n",
    "    for i, batch in enumerate(data_iterator):\n",
    "        data = batch.data[0].as_in_context(ctx).reshape((-1,784))\n",
    "        label = batch.label[0].as_in_context(ctx)\n",
    "        output = net(data)\n",
    "        predictions = nd.argmax(output, axis=1)\n",
    "        acc.update(preds=predictions, labels=label)\n",
    "    return acc.get()[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0. Loss: 0.23931532364, Train_acc 0.946860730594, Test_acc 0.945262738854\n",
      "Epoch 1. Loss: 0.14545513, Train_acc 0.955507990868, Test_acc 0.948669628594\n",
      "Epoch 2. Loss: 0.0970559549169, Train_acc 0.961448820396, Test_acc 0.956427087772\n",
      "Epoch 3. Loss: 0.0713375280292, Train_acc 0.965760559361, Test_acc 0.961963429692\n",
      "Epoch 4. Loss: 0.0562312310473, Train_acc 0.969003995434, Test_acc 0.966091304827\n",
      "Epoch 5. Loss: 0.0471335310744, Train_acc 0.971729927702, Test_acc 0.969227183949\n",
      "Epoch 6. Loss: 0.0375632641638, Train_acc 0.973980756686, Test_acc 0.971890330013\n",
      "Epoch 7. Loss: 0.0306285313321, Train_acc 0.975852597032, Test_acc 0.974077521734\n",
      "Epoch 8. Loss: 0.0244164425957, Train_acc 0.977409944191, Test_acc 0.975897513177\n",
      "Epoch 9. Loss: 0.0171164385061, Train_acc 0.978745719178, Test_acc 0.977425526868\n"
     ]
    }
   ],
   "source": [
    "epochs = 10\n",
    "moving_loss = 0.\n",
    "\n",
    "for e in range(epochs):\n",
    "    train_data.reset()\n",
    "    for i, batch in enumerate(train_data):\n",
    "        data = batch.data[0].as_in_context(ctx).reshape((-1,784))\n",
    "        label = batch.label[0].as_in_context(ctx)\n",
    "        with autograd.record():\n",
    "            output = net(data)\n",
    "            cross_entropy = loss(output, label)\n",
    "            cross_entropy.backward()\n",
    "        trainer.step(data.shape[0])\n",
    "        \n",
    "        ##########################\n",
    "        #  Keep a moving average of the losses\n",
    "        ##########################\n",
    "        if i == 0:\n",
    "            moving_loss = np.mean(cross_entropy.asnumpy()[0])\n",
    "        else:\n",
    "            moving_loss = .99 * moving_loss + .01 * np.mean(cross_entropy.asnumpy()[0])\n",
    "            \n",
    "    test_accuracy = evaluate_accuracy(test_data, net)\n",
    "    train_accuracy = evaluate_accuracy(train_data, net)\n",
    "    print(\"Epoch %s. Loss: %s, Train_acc %s, Test_acc %s\" % (e, moving_loss, train_accuracy, test_accuracy))    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Conclusion\n",
    "\n",
    "Now let's take a look at how to build convolutional neural networks."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For whinges or inquiries, [open an issue on  GitHub.](https://github.com/zackchase/mxnet-the-straight-dope)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}