MicroGrad.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MicroGrad\n",
    "\n",
    "A tiny Autograd engine"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![awww](puppy.jpg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The tiniest Autograd engine. It's so cute!\n",
    "\n",
    "class Value:\n",
    "    \"\"\" stores a single scalar value and its gradient \"\"\"\n",
    "    \n",
    "    def __init__(self, data):\n",
    "        self.data = data\n",
    "        self.grad = 0\n",
    "        self.backward = lambda: None\n",
    "    \n",
    "    def __add__(self, other):\n",
    "        other = other if isinstance(other, Value) else Value(other) # attempt to wrap if given an int/float/etc\n",
    "        out = Value(self.data + other.data)\n",
    "        \n",
    "        def backward():\n",
    "            self.grad += out.grad\n",
    "            other.grad += out.grad\n",
    "            self.backward()\n",
    "            other.backward()\n",
    "        out.backward = backward\n",
    "        \n",
    "        return out\n",
    "    \n",
    "    def __radd__(self, other):\n",
    "        return self.__add__(other)\n",
    "        \n",
    "    def __mul__(self, other):\n",
    "        other = other if isinstance(other, Value) else Value(other) # attempt to wrap if given an int/float/etc\n",
    "        out = Value(self.data * other.data)\n",
    "        \n",
    "        def backward():\n",
    "            self.grad += other.data * out.grad\n",
    "            other.grad += self.data * out.grad\n",
    "            self.backward()\n",
    "            other.backward()\n",
    "        out.backward = backward\n",
    "        \n",
    "        return out\n",
    "    \n",
    "    def __rmul__(self, other):\n",
    "        return self.__mul__(other)\n",
    "    \n",
    "    def relu(self):\n",
    "        out = Value(0 if self.data < 0 else self.data)\n",
    "        def backward():\n",
    "            self.grad += (out.data > 0) * out.grad\n",
    "            self.backward()\n",
    "        out.backward = backward\n",
    "        return out\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return f\"Value(data={self.data}, grad={self.grad})\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A neural networks \"library\" :D on top of it! I'm dying\n",
    "\n",
    "class Module:\n",
    "    def zero_grad(self):\n",
    "        for p in self.parameters():\n",
    "            p.grad = 0\n",
    "\n",
    "class Neuron(Module):\n",
    "    \n",
    "    def __init__(self, nin, nonlin=True):\n",
    "        self.w = [Value(random.uniform(-1,1)) for _ in range(nin)]\n",
    "        self.b = Value(0)\n",
    "        self.nonlin = nonlin\n",
    "    \n",
    "    def __call__(self, x):\n",
    "        act = sum([wi*xi for wi,xi in zip(self.w, x)], self.b)\n",
    "        return act.relu() if self.nonlin else act \n",
    "    \n",
    "    def parameters(self):\n",
    "        return self.w + [self.b]\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return f\"{'ReLU' if self.nonlin else 'Linear'}Neuron({len(self.w)})\"\n",
    "\n",
    "class Layer(Module):\n",
    "    \n",
    "    def __init__(self, nin, nout, **kwargs):\n",
    "        self.neurons = [Neuron(nin, **kwargs) for _ in range(nout)]\n",
    "    \n",
    "    def __call__(self, x):\n",
    "        out = [n(x) for n in self.neurons]\n",
    "        return out[0] if len(out) == 1 else out\n",
    "    \n",
    "    def parameters(self):\n",
    "        return [p for n in self.neurons for p in n.parameters()]\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return f\"Layer of [{', '.join(str(n) for n in self.neurons)}]\"\n",
    "\n",
    "class MLP(Module):\n",
    "    \n",
    "    def __init__(self, nin, nouts):\n",
    "        sz = [nin] + nouts\n",
    "        self.layers = [Layer(sz[i], sz[i+1], nonlin=i!=len(nouts)-1) for i in range(len(nouts))]\n",
    "    \n",
    "    def __call__(self, x):\n",
    "        for layer in self.layers:\n",
    "            x = layer(x)\n",
    "        return x\n",
    "    \n",
    "    def parameters(self):\n",
    "        return [p for layer in self.layers for p in layer.parameters()]\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return f\"MLP of [{', '.join(str(layer) for layer in self.layers)}]\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(1337)\n",
    "random.seed(1337)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x10c5223c8>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU4AAAEyCAYAAACVsznTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xd4VFX6wPHvmZJJJiGQhNB7C0iVqoCKAiIgAiqCiqKyshYsu7qKrq4/62Jd7IoKCqKAsDakF3UVpUqXEppAaCFAyiSZdn5/ZIgZkkDKJHcm836eJ09y75x775tJ8ubce5rSWiOEEKLkTEYHIIQQoUYSpxBClJIkTiGEKCVJnEIIUUqSOIUQopQkcQohRClJ4hRCiFKSxCmEEKUkiVMIIUrJYnQAZVGzZk3dpEkTo8MQQlQx69atS9VaJ56vXEgmziZNmrB27VqjwxBCVDFKqf0lKSe36kIIUUqSOIUQopQkcQohRClJ4hRCiFKSxCmEEKUkiVMIIUpJEqcQQpSSJE4hhCglSZxhzOPxsmfPSY4ezTQ6FCFCSkiOHBLld+RIJpdf/gl//HGanBw3JpOiRg0bEyf2Y+zYzkaHJ0RQkxpnmLrttq9ITk7D4XDh9Wrcbi+pqdncf/9CFi/ebXR4QgQ1SZxhasOGI7jd3kL7HQ4XX3+93YCIhAgdkjjDVPPmcZhMqtB+q9VEzZp2AyISInRI4gxTH388jISEKKKjrfn7IiJMJCTYGT++u4GRCRH8pHEoTLVsmUBy8v2sW5fC4cMZ7N17ipiYCEaP7kBCgtQ4hTgXSZxhLDbWxuWXN83f3rEjldWrD9G6dU2aNo0zMDIhgpvcqocJrTVvvLGKZs1ep3nz1/nww/V+r0+c+BMXXvg+o0bNpW3bd5g2bUOpr3HqVA4//rifLVuOobUOVOhCBB0Vir/gXbt21TIDfOl8+OF6HnhgIQ6HCwC73cqUKdcwcmQ7du9Oo337d8nOdueXj4y0cOTIQ1SvHlmi8//222GuuGIaWmtcLi/XXdeGTz4ZhlKFG6CECFZKqXVa667nKyc1zjAxZcpv+UkT8rodffxxXq1y375TRESY/cpbLCYOHy75iKLrr/+CU6dyOH06F4fDxeefb2Hu3N8DE7wQQUYSZ5iIiYkoYp8NgDZtEnG5/Pt0mkzQqFH1Ep9///5Tfttut5f33ivfXUFGRi4rVx5gx47Ucp1HiECTxBkmnnnmcuz2P7seRUdbeeKJSwCoV68a06cPJyrKQnS0ldhYG998c6Nf+fOJjCzcznjwYHqZ49206ShNmrzOwIEzuPDC97n99q/kuakIGtKqHiYuuqgBK1fewdSpGzCZFHfe2Zk2bf5cBfXaa9tw4sQjHDmSSb161bDZSverceWVzfnyS/8RRx061C5zvCNGfEFaWnb+9hdfbGPo0NYMG9a6zOcUIlAkcYaRjh3rMGnSVcW+HhVlLXM3pLffHsTPP/9BZqYL0NjtEbz8cv8yRpr33LWg3FyP3LKLoCGJUwRE3brV2LHjPhYs2IXWcNVVLYiPj8Lt9vLDD/vIyHDSs2dDatWKLtH5WraMZ9u245y5O7fZzLRrV6sCvwMhSk4SpwiYGjUiufHG9vnbTqeHPn0+ZvPmY/nj4n/44TY6dapz3nPNmXMDffp8jMPhwun0MHZsZwYNallhsQtRGpI4RYX58MP1bNhwxK9/6JgxX7Fx413nPbZ165rs2/cgu3adID4+ivr1YysyVCFKJSCt6kqpKUqpY0qpLcW8rpRSbyilkpVSm5RSnQu8NkYptcv3MSYQ8YjgsHfvKb+kCcW3tDudnkKt5pGRFtq3ry1JUwSdQHVH+hgovtUBBgItfR/jgHcBlFLxwFNAD6A78JRSSgZJVxEXX9zAb/Ylq9VE9+71/MocOZJJ166TiYp6nujoF5g69bfKDlOIUgtI4tRa/wiknaPIUGCazvMrUEMpVRcYACzRWqdprU8CSzh3AhYhZPjw1tx3X3csFhMREWbatq3FtGnD/cpce+0sNm48iteryc52M378AlavPmRQxEKUTGU946wPHCiwfdC3r7j9hSilxpFXW6VRo0YVE6UIKKUU//53P5588jKys13Ex0cVGru+Zk2K30z0breXn3/+g+7di/w1ECIoVNbIoaJmetDn2F94p9aTtdZdtdZdExMTiyoigpTdbiUhwV7khB9xcf6TiFitJurUiams0IQok8pKnAeBhgW2GwAp59gvwsTUqUOx2/OGesbERNC5c11GjGhrdFhCnFNl3ap/A4xXSs0kryHotNb6sFJqEfBCgQahK4HHKimmkJSSksGmTUepX78a7duXfUhjsBg8uBXr1/+Vn376g5o17Qwe3AqLpWT/z7dvT2XVqoPUqRND//7Ni1xDSYiKEJDEqZT6HOgD1FRKHSSvpdwKoLV+D5gPDAKSAQdwu++1NKXUs8Aa36me0Vqfq5EprC1YsIsRI77AYjHhcnkZN64L//nPAKPDKrekpJokJdUs1TFz527j1lu/RCmFUoo+fRrz9dc3SvIUlUImMg4RXq+mevWJZGY68/dFR1tZuvRWLrqogYGRVT6tNbGx/u9FTIyV22/vxOzZ28jN9XDjje14/fWrsFrN5ziTEP5kIuMqJj09l9xc/87kJpNiz56TBkVkHJfL6zcp85l977+/nqNHszh1KoePP97AhAnLDIpQVHWSOENE9eo24uKi/Pa53d5ST93m9Wpef/1XrrxyOnfc8TUpKRmBDLNSRESYad26pt9tudvtxen05G9nZ7uZO3ebEeGJMCCJM0QopVi48GZq1sxbC91mM/Paa1eWesagv/1tIY8/vpwlS/YwffpGLrzwfb95L0PF/Pk3kZSUgNmsiIqyMHBgi0KNSiVdL0mI0pJnnCHG5fJw4EA6iYl2qlWzlepYr1cTGfmc3zIZ0dFW3n57EGPGdAp0qJUiO9tFZKSFo0ez6NDhXU6fzsXj8WKzWZg370a/5Y+FOJ+SPuOU2ZHOw+vVvPbaL8yevZWEBDsvvtivXDObl5fVaqZZs7IP5z/7/6TW4PGE3j/PM6Ki8sbC16kTw5Yt9zB9+kYcDhdDh7Y29OckqjZJnOfx+OPLePPN1TgcLpSCn376g40b7ypX8jKKyaQYPbo9s2dvxeFwYzIpbDYzgwdXjXkua9WK5qGHehodhggDkjjP47331ua34GoNublu5szZxiOP9KrQ6+7de5I5c7ZhMilGjmxHWlo2S5fuoXp1G6NGtSM6uvCqlSXxwQfX0KBBLAsWJFO/fjVefXUAtWvLEEchSkMS53mc3aFaqcL7Am3z5qP07DmF3Fw3SsG//rUCrzfvsYHFYuKll1ayfv24MiVPi8XEs89ewbPPXlEBkZdOZqaTjIxcateOkY7rIqRIq/p5PPRQz/xlck0mRVSUlVGj2lXoNSdMWEpWlhOXy4vT6cXhcJOT48bp9OBwuDhw4DQff7yhQmOoaM888wMJCS/SrNkbJCW9xYEDp40OSYgSkxrneTz+eG/q1o1h9uytJCba+b//60ODBhU7I3lqqqNQI05BubluUlMdFRpDRVqyZDcvvfQzTqcX8LJ370muv342q1bdaXRoQpSIJM7zUEpxxx0XcscdF1baNUeMaMuWLcfzn62aTAqTSeXPWxkZaaFfv2aVFk+grV2bQk7On6OgPB7Npk3HDIxIiNKRW/Ug9Pe/X8x993X3jRaK5LHHejNwYAtsNjPx8VG8//4QevUK3cmcGzeuQWSk///sunWlgUqEDukAH0BaaxYsSGbbtuO0bl2TwYNbFjl5b7jzeLxcc83n/PDDfsxmE1prFi++JewmKxHBRzrAG+D++xcwdeoGnE4PERFmbrmlA+++e7XRYQUds9nEvHk38fPPBzh5Mpvu3etLlygRUqTGGSD795+ideu3/Z7dRUZa2LLlbpo3jzcwMiFEScm0cpUsLS2biAj/uR8jIsycOBF6E2gIIc5NEmeAJCXVLJQ4rVYTTZrU4Pbbv6J27Vdo0+YtVqzYa1CEoS0lJYMtW4751eiFMIokzgCx2618//0YkpISsFhMtGwZz/LlYxg/fj4zZ27l2LEstm8/wdVXf87vvx83OtyQ8ve/L6JZs9fp2fMjGjeexPbtqUaHJMKcNA4FUNu2tdi+fbzfvm+/3elXS3K7vSxcmEybNrLEcUksWLCLyZPXkZvrITfXQ2amk2uvncW2bfcaHZoIY1LjrGCRkf637xaLiZiYsk3QEY62bDnmN7O71pCcLOv5CWNJ4qxgL77YP3+se0SEmVq17IwcWbFj3auSVq0SCj07btSoukHRCJFHbtUr2LhxXWjatAYLFiRTq1Y0d93VldjY0s3cHs6uuSaJG25oy8yZW7BazZjNirlzb6jw62qtmTjxJ15+eSVer2bcuC5MnNhPZnESgPTjFCFix45UTpzIpl27WpXyj+fjj3/j3nsX5M8XYLdbeeKJS3nssd4Vfm1hHBk5FKL27TvF3LnbUEoxcmRb6tev2JmYQkVSUs1Kvd4XX2zzW4LY4XAxd+42SZwCkMQZVLZuPcbFF3/kWz9d8cwzP7Bu3TgZeWSAmjWjMZnA++e6diQkRBV/gAgr0jgURB59dCmZmU6czrw1wjMynPzrXyuMDissPfXUZcTGRmKxmPKXIH7ppf5GhyWChCTOIHL2BMZer+b48dCdsDiUNWsWx733dkMpfDM4wVdfbTc6LBEkApI4lVJXKaV2KKWSlVITinj9P0qpDb6PnUqpUwVe8xR47ZtAxBOqrr/+gvyuS5DXIHHddRcYGFH4Ono0k1dfXelbvsRDTo6biRN/Zt++U+c/WFR55X7GqZQyA28D/YGDwBql1Dda621nymit/1ag/H1AwenUs7XWncobR1Xw979fzPHjWbz33jqUgr/97SLGjetsdFhh6fDhTCIiLOTk/Nn53mYzk5KSQZMmNQyMTASDQDQOdQeStdZ7AJRSM4GhwLZiyt8IPBWA61Y5JpPixRf78+KL8izNaC1aFG6Q83g0rVtXbuu+CE6BuFWvDxwosH3Qt68QpVRjoCmwvMDuSKXUWqXUr0qpYQGIR4hyi4mJYMGCm4mLiyQiwkxsrI1vvhlFfLy0rIvA1DiLGkpRXK/6UcAcrbWnwL5GWusUpVQzYLlSarPWenehiyg1DhgH0KhR6K63I0JHz54NOXHiEU6ezKFGjUgZNSTyBaLGeRBoWGC7AZBSTNlRwOcFd2itU3yf9wDf4//8s2C5yVrrrlrrromJMrOQqBxKKeLjoyRpCj+BSJxrgJZKqaZKqQjykmOh1nGlVBIQB/xSYF+cUsrm+7om0Ivin40GrcOHM5g7dxtLluzG4/Ge/wAhREgr96261tqtlBoPLALMwBSt9Val1DPAWq31mSR6IzBT+w+ObwO8r5TykpfEJxZsjQ8Fa9Ycom/faUDelGedOtVh+fJbsVrN5zlSCBGqZJKPMsrOdjFjxmYee2wZqal/dlK3261MmjSAO+/sYmB0QoiykEk+KlBOjpvu3T9kz56TfhNBQN5kEAcOpBsUmRCiMkjiLINZs7awd2/hpAl5Nc4ePYrsjSWCTGqqgylTfiMjI5drrkmiWzf5uYmSkcRZBidP5uB2F24EslhMPPJITwYPbmVAVKI0jh/Pon37dzl5MgeXy8Nrr/3C7Nkj5GcnSkQSZxlccUVTv+4pERFmevVqyPz5NxMZKW9pKHj//XWkpWXjcuX9A3Q43Dz44CJJnKJEZHakMujQoTazZl1PnToxREVZ6Nu3Kf/970hJmiEkr6bpf9eQkZFrUDQi1MhfehkNGZLE4cNJRochymjYsCTee28NDkfe0s1RURauu66NwVGJUCE1ThGWLrmkMVOnDqNhw1gSEqK49daO/Oc/VxkdlggR0o+zlLZuPcaGDUdo0qQGvXrJmHkhqhLpx1kBPvroN+67bz4WiwmvVzN6dAfee+9qo8MKW1prPvhgPbNmbSE+3s6zz14u076JSiE1zhLKznYRF/ciubl/Tuxkt1v58cfb6NKlXqXGIvI8//z/eOGF/+FwuFAqbyq4TZvulomGRZmVtMYpzzhL6MSJ7EIz5FgsJg4dyjAoIjFp0i/5gxC0huxsNzNnbin3eXfvTuOtt1bz0UfrSU+XlnZRmNyql1DdujHExtrIznbn73O7vXTsWNvAqMJbUTdLXm/57qBWrTpI377T8Hi8mEwmnn76B3777a8kJNjLdV5RtUiNs4TMZhNLltxCvXrVsFhMREdbmTnzOho3lttCo4wf3z1/cTulIDLSwqhR7cp1znvumU9WloucHA8Oh4sjRzKZNOnXQIQrqhCpcRYjJSWDp5/+gZSUdIYMSeLOOzvTvn1tDh78GxkZTmJiImRyW4M99dRlJCREMXPmVuLjo3jhhSto1iyuXOcsONMVgMvl5fDhzHKdU1Q9kjiLkJaWzYUXvkdaWjZut2b58n3s2XOSiRP7oZQiNtZmdIiCvNnZ77uvB/fd1yNg5xw4sAXTpm3MfyRjt1sZPLhlwM4vqga5VS/Cl1/+TmamC7c773mZw+Fi0qRfCcUeCKJ0Jk26iiFDkrBa8x7HPP10H4YPlxFFwp/UOIvgdnsLJcnyNjqI0BAZaWHWrOuNDkMEOalxFuHqq1thtZpRvkeYdruV0aM7oJQ80xRCSOIsUv36sfz661iuvLI5HTvW5sEHe/D++zJCSAiRR27Vi9GmTSILF442OgwRBJxODx6Pl6goq9GhiCAhNU4hiqG15r775hMd/TzVqv2bwYM/Izu78HIpIvxI4hSiGO+/v44pUzbgdms8Hs3y5Xt56KHFRoclgoAkTiGKsWzZHr8F+XJy3CxfvtfAiESwkMQpRDGaNIkjIsKcv20yKRo1qm5gRCJYSOIUohiPP96bBg2qUa1aBDExEVSvbuPttwcZHZYIAtKqLkQx4uKi2Lz5HhYtSsbp9NC3bzNq1pRZkoQkTiHOyW63ypBLUYjcqgshRCkFJHEqpa5SSu1QSiUrpSYU8fptSqnjSqkNvo+/FHhtjFJql+9jTCDiEUKIilTuW3WllBl4G+gPHATWKKW+0VpvO6voLK31+LOOjQeeAroCGljnO/ZkeeMSQoQfrTW7Fy3i+LZt1GzdmhYDB1bIHBOBeMbZHUjWWu8BUErNBIYCZyfOogwAlmit03zHLgGuAj4PQFxCiDCz6MEHWf/RR3hdLkxWKx3HjGHw228H/DqBuFWvDxwosH3Qt+9s1ymlNiml5iilGpbyWJRS45RSa5VSa48fPx6AsEUgJS9cyLfjxrHk0UfJOHzY6HBEGDr9xx+smzwZV1YWHqcTV1YWG6ZM4eSePQG/ViASZ1H14LMnr/wWaKK17gAsBT4pxbF5O7WerLXuqrXumpiYWOZgReBtmDqV2dddx/oPPuDX117jvY4dyTx61OiwRJhxpKZijojw22eOiMBx4kTArxWIxHkQaFhguwGQUrCA1vqE1vrMOqsfAF1Kemwoyc52sW5dCrt2nQir2eKX/fOfuBx5a/V43W5yT59m47RpBkclwk3N1q0xWf1nsDJZLCS2CXx3skAkzjVAS6VUU6VUBDAK+KZgAaVU3QKb1wC/+75eBFyplIpTSsUBV/r2hZzdu9No3vwNLr/8Ezp2fI+bbpobNrPGu3Ny/LY9bnd+Ig01J09m8913O1mxYi9ut/e85XfvTmPs2K8ZPnwms2dvrYQIw0P2yZP8NHEiSx55hH0//FCiY6x2O7d9/z3xLVuizGbimjfn1uXLiYiJCXh85W4c0lq7lVLjyUt4ZmCK1nqrUuoZYK3W+hvgfqXUNYAbSANu8x2bppR6lrzkC/DMmYaiUHPzzf/l6NGs/GT57bc7+eyzzYwe3cHgyCpe+5tv5rcpU3D7kqU1MpLWw4YZHFXp7diRSs+eU3C7PXi90Lp1TX788bZi5+H844/TdOkymYyMXLxeWLx4D8ePO7j33m6VHHnVknPqFO916EDWsWN4nE7WvP02V0+eTIebbz7vsbXateO+nTsrPMaA9OPUWs/XWrfSWjfXWj/v2/cvX9JEa/2Y1rqt1rqj1vpyrfX2AsdO0Vq38H1MDUQ8Rtix44RfDTMry8XWrccMjKjyDHjtNbrdfTfVGzemdocO3DhvHnU6djQ6rFIbO/YbTp7MJj3dSWamky1bjvHWW6uLLT99+kayslx4fRVTh8PF88//WEnRVl0bp03DkZqKx+kEwOVwsPihhwyOyp8MuQyQpKQE1qxJyU+e0dFW2ratZXBUlcNstXLlK69w5SuvGB1Kuezbd4qCj6Zzctzs2lX8DZDT6Sn0OKYkt/fi3JyZmXhc/hNGB9ujHxlyGSCffXYddepEExtrIyrKwjXXJHHTTe2NDkuUQo8eDYiI+PNPwm630qtXw2LLjxzZjqgoi1/5ceO6FFtelEyLgQMx22z525YgfPSjQrH1t2vXrnrt2rVGh1FITo6b338/TrVqNpo3j6uyq2K6HA48TieRNWoYHUpApaVlc+WV09my5Rher2bs2At5553B5/w5rlp1kEceWcrp0zmMGtWORx7phclUNX/ulWnXggUsuO8+ck+fptWQIQx+5x0skZEVfl2l1DqtddfzlpPEKUpKa83CBx5g7bvvglLU79aNm+bPJ7J61ZncV2vN8eMOIiMtxMbazn+AqFJKmjjlVl2U2Kbp0/ltyhS8bjdel4uUdeuYd9ddRocVULNnb+Xmm//LmDFfsn69jIASRZPGIVFi+374AVdWVv62JzeXAz//bGBEgfXRR+u5//6F+esMLVmyh19+GUv79rUNjkwEmypf4/R6NWvXpvDjj/vJzHQaHU5Ii2/e3P85k1JUb+jfeOLOyWH3kiUkL1yIMzOzkiMsnxdf/NlvcbasLBcffLDewIhEsKrSNU6Xy8PAgTP49deDmM0moqIsrFw5lmbN4owOLST1eOABts6axck9e1AmE8psZsiHH+a/nnPqFB/26EHG4cMopYioVo07V6+mWr16BkZdckWN9AqX0V+idKp0jfPdd9eycuUBsrJcpKfncvy4g9tu+8rosEJWRHQ0d65Zw4g5cxj68cfct3On3zjg7596ilP79uHMyCA3PZ3Mo0dZ8MADBkZcOg891BO7/c9RQna7lb/8pbOBEYlgVaVrnNu2HSc7252/7fVqkpNDckRn0DBHRNBiwIAiX0vduTN/tAeAdrtJS06urNDK7e67uxIVZeGjj34jJsbKU0/1oVOnOkaHJYJQla5xdutWz68GYbGY6NhR/hAqSuNLLsFq/3MVSEtkJA179jQwotK77bZO/O9/t7NgwWguuqiB0eGIIFWlE+ftt1/I8OGtsdnMREdbadYsjqlTh/qVSU5O45lnfuCZZ36Q2mg59XrkEVoMGoTJasUcEUGDnj258uWXjQ5LiIALiw7wKSkZOBwumjatgdn85/+KzZuP0rPnFLKz81pSo6KsrFx5h3Q/KafstDS8Hg/2mjWr7OgpUTVJB/gC6tWrRosW8X5JE+Bf/1pBVpYTj0fj8Wiyspw8+eQKg6KsOqLi44lOTJSkKaqsKt04dD5paTl+s+FoDSdP5hR/gAhrCxcms2BBMnXqRHPPPd2oXr3ix05XNVprMg4dwuNyUaNxY5QpNOtuYZ04b765PWvXpuR3eo6OtsqMRqIQj8fL3/++iPfeW4fT6cFmM/PBB+vZtOluYmIizn8CAYDH5WLW8OHsXbYMlKJWu3bcunQptthYo0MrtdBM9wFy552defLJS6ldO5pataJ54olLGTdO+u2JP7lcHq64YhpvvLEap9MDQG6uh2PHspgzpyQrYIszfn7pJfYuX447Jwd3djZHN21i4YMPGh1WmYRljfOTTzbyn//8gtls4p//vIQjRx42OiQRpKZO3cDatYXXD/R4tAzhLaVDv/6KOzs7f9uTm8uhNWvOcUTwCrvE+emnm7jnnu/yb89vueVLbDYzgwe3MjgyEYx2707zG79+hsmkuPLK5gZEFLoS27Zl99KleHyL+5ms1gpZgbIyhN2t+jvvrPH7Q3A4XLz7rsztKYp20UUNiI72X6zNbreycOHNtGqVYFBUoenSJ54gsU0bImJiiKhWjWr16zPwjTeMDqtMwq7GGRFhLrQvMjLs3oYyObJhA1+MGMGp/fuJb9mSkf/9LzWTkowOq0ING9aae+7pxqRJv2IyKVq2TGDp0luoXTvwS85WdRExMdy5ejUpa9fidbup26UL1qgoo8Mqk7DoAF/QsmV7GDLk8/wx7Ha7lRUrxtC9e/1Ahljl5KanM6lJE3JOnszboRTRtWrx4P79WGxVf6b0zEwnDoeLxES79E8NgD9++on548eTnZZGqyFDGPDaa0Hxe1TSDvBhV9Xq27cZixffwjvvrMFiMfHAAz3o0iU0pj0z0tHNm9HeAis4ao3L4SAtOZlabdsaF1gliYmJkK5HAZK6fTufDhiQv3LlhqlTcWVmMuyTTwyOrOTCLnEC9O7diN69GxkdRkiJio/He9aSrR6nk6j4eIMiEqFq53ff+S3/687OZtucOedMnBkpKSz5xz84uXcvzfr149Inn8RstRZbvqKFZeIUpZfYpg0XjBjBtjlz8OTmYrbZ6DJuHNXq1jU6NBFirFFRmCwWv3/E5ojia/M5p08zuUsXslJT0W43RzZs4MTOnVw/c2ZlhFskSZyixIZOnUrrYcM4sXMntdq3p+XAgUaHJEJQuxtv5MfnnsPhW/TPardzxXPPFVt+z9KlOLOy0O68dokzNVR3Tk6lLBlcFEmcosSUUrQeNszoMESIi4qL4+5Nm/j19ddxHDtGqyFDaHX11cWWD8bGOEmcZeTxeDl6NIv4+CjpziREKdlr1uSKZ58tUdlm/foRERODOzsbr9uN1W4naehQw2qbEKAO8Eqpq5RSO5RSyUqpCUW8/nel1Dal1Cal1DKlVOMCr3mUUht8H98EIp6KtnnzUerXf40WLd6gRo2JfPLJBqNDEqLKssXGMm7dOtrfdBONLrmEno88wvBp0wyNqdz9OJVSZmAn0B84CKwBbtRabytQ5nJgldbaoZS6G+ijtR7pey1Ta12q3sTl6cdZXlprGjT4DykpGfn7oqIsrF39MSFPAAAgAElEQVQ7jgsuSDQkJiFEYFTmRMbdgWSt9R6ttROYCfitT6G1XqG1dvg2fwVCdjGXU6dySE3N8ttnsZjYuPGIQREJISpbIBJnfeBAge2Dvn3FGQssKLAdqZRaq5T6VSlVbMuDUmqcr9za48ePly/icqhePRKLxX/YpteradSoukERCSEqWyASZ1FNXkXe/yulRgNdgYIreDXyVY1vAiYppYqcckZrPVlr3VVr3TUx0bhbYpNJMWPGtdjtVmJjbdjtVkaP7kCvXtKhXpSN1pqjRzPz174SwS8QzcEHgYYFthsAhSYwVEr1A/4JXKa1zj2zX2ud4vu8Ryn1PXAhsDsAcVWYYcNas3XrPWzYcIQGDWLp2lWGbIqiZWU5efjhxfz88wFatkzgjTeuon79P2c8P3DgNH37TuOPP07j9WqefvpyHnust4ERi5IIROOQhbzGob7AIfIah27SWm8tUOZCYA5wldZ6V4H9cYBDa52rlKoJ/AIMLdiwVBQjG4eEKCmtNZdd9jFr1hwiJ8eDxaKoXTuG7dvH549779btA3777TAeT97fod1uZd68G7n88qZGhh62Kq1xSGvtBsYDi4Dfgdla661KqWeUUtf4ir0MxABfnNXtqA2wVim1EVgBTDxf0jTS778f57vvdrJnz0mjQxEh4NixLFavzkuaAG63JiMjl5Ur/2wS2LTpaH7SBHA6PaxefajSYxWlE5Ce21rr+cD8s/b9q8DX/Yo5biUQEqujPffcj/z73//DYjHjcnl4993BjBnTyeiwgtqGqVNZ9vjjuHNyaDtyJAPffNPQiRkqm9ls4uwbOq3zemGcUadONH/8kZ6/bbOZady4RmWFKMoo7GaAL4tdu07wwgv/w+Fwk56eS3a2m7vu+o709NzzHxymkhctYv748WQeOULOqVNsnDaNJY88YnRYlapmTTtDhrTCbs/7Z2GzmWnQIJZevf5sEvjss+uIiYkgNtZGTEwEl13WmBEjLjAqZFFCMlawBPbvP01EhDl/8mPIqzUcOZJJbKzxk68Go+1ffpk/3yLkTczw+9y5XPWf/xgYVeWbOfN6Xn55JT/99AdJSQk89dRl2Gx//tn16tWIHTvGs3r1IeLjo+jduxEmU/CNzQ5WWmuSFywgdft2Ei+4gOYDBlTK2HZJnCXQpk1NXC6P3z6TSdGwYeitB11a2usFpUr9yxgVH583dZj7z382kdXDr6+rxWI6byt5vXrVGDasdSVFVLXMHz+ejZ98gtflwmS10nnsWK56/fUKv27Y3qq73V7efHMVY8Z8xSuvrMxfM7so9evHMn36tURFWbDbrVSvbmPevBuJiqq6z+s8Tidf3nILz9lsPB8VxdIJEyhND4weDzxAZHw85ogIlNmMxW6vlF9oYTxnZia56ennL1hOJ/fsYcOUKbiysvA4nbiyslg3eTKn9u+v8GuHZY1Ta831189myZI9OBwuoqIszJ+/i6VLby32Nunaa9uQlvYoR49mUrdutSIXfatKlj/xBNvmzs2rMbrdrH7zTeJbtKDzX/5SouNjatfmni1b2DhtGi6Hg6RrrqFOx44VHLUwktft5ssxY9g2ezYAzQcM4IY5cypsFiNHairmiAjcvuWGIW9C5OwTJ6jRuPE5jiy/sKxx7tt3isWLd+cvE5yd7Wb16kNs2XLsnMdFRlpo3LhGlU+aALvmz8ednZ2/7XI42DlvXqnOEZ2YSM+HHuKyJ5+UpBkGVr7yCju++gqv243X7WbvsmUse/zxCrte4gUXYLL41/1MVisJlbDyalgmzpwcd6GapcmkZMhbATF160KB55omi4XYBiE7N4uoBPtWrPBvEMzJYd8PP1TY9SJiYhizYgVxzZujzGbiW7ZkzIoVRERHV9g1zwjLW/WWLROoXz+WvXtP4nJ5MZsVNWpE0qFDbaNDCxoDX3+djy6+OO9WXSls1apx6ZNPGh2WCGJxLVpgWrEify0hZTYT17RiR0DV7tCB+5OTK/QaRQm7ddXPOHYsizvv/IYNG47Spk1NPvzwGho0qPqt5KWRfugQyQsWYLJYSBo6lKi4OKNDEkHMceIEH3TrhiM1FYCI6GjuXLMmpO5USjrkMmwTpxAi8FwOB3uXL8fr8dCkT5+Q64JW0sQZlrfqongel4tNn35K+oED1O/RgxYDBpT5XBkpKeSmpxPXvHlYDbUMZ1a7/ZwLr1UVkjhFPq/Hw/T+/UlZswZXdjbWqCh6P/44l/7zn6U6j9aaeX/9KxunTcNktWJPSOD2H3+keiOZs1RUDWHZqi6Ktm/FCg6vW5fXMqo1LoeDH55+Gndu6cbkb509m82ffYYnNxdXZibpBw8y96abKihqISqfJE6RL+fUKb8uSAAo5dfFpCSObNiAK+vPdZm0x8OxLVsCEaIQQUESp8jXsGdPCs6DpiwWaiYlEVmjdNOc1WzdGmvBvnRKEde8yBVRhAhJkjhFvmr16nHLkiXEtWhBREwMjXr14pbFi0s9wUeH0aNp1rcvVrsdW2ws9oQErpsxo4KiFqLySXckUSG01hzdtInc9HTqdOqErVo1o0MKapMnr+P//u97nE4Pt9/eiYkT+2E2m9i//xS33fY127en0q5dLT7+eKjfmkUisKQ7kjCUUkrGp5fQ119v529/W5Q/d8I776wlOjqCRx/tRe/eUzh8OBOPR3P8eBa9e09lx47xYTFfQjCTW3UhDDZr1tb8pAngcLiYOXMLW7Yc4/Tp3Pw1iTweTWqqg507TxgVqvCRGud5pKVl8+abqzh6NItBg1py9dWtjA5JVDHx8VGYzcpv0bbq1SOx261++wA8Hm/+UhzCOFLjPIfTp3Po1Ok9XnjhJ959dy0jR87hjTdWGR2WqGIeeaQXsbE2rFYTJlPeEsGvvnolF1yQSJ8+TfITpd1uZeDAljRtKou5GU0ah87hgw/W8eCDi/xuo6pViyA9/bEKv3Yo2714MUc2biS+eXNaDx9eKWvAhLpDh9KZNm0jTqeH6667gHbtagF5KxV88ME6Nm48Spcudbnjjgsxm6W+U1GkcSgAHA4XHo/Xb9+5ltgQsOzxx1n1xht4nE7MEREkDR3KtZ9+KsnzPOrXj+Wxxy4ptN9iMXH33d0MiCiwvB4PSimUqWok/arxXVSQgQNb+q2BHRVlkUW1zsGRmsovr76KKysLr8uFKyuLHV99xdFNm4wOTVQS7fWScfgwzsxMIG/SmK/GjOH5yEies9mYf999eQsAhjipcZ5Dq1YJLFo0mnvvnU9qqoNBg1ry+utXGR1W0Mo+eRJzRAQepzN/n8lqJfuEtAKHg/RDh5jWty+n9u9Hezxc8vjjeD0ets2Zk7/a6YYpU4hv0YKLHnjA4GjLRxLnefTq1YgNG+4yOoyQUKNJE2zVq+PMyvIbulmnUycDoxKVZc4NN5CWnIz25D3OWvnKK8TUres314HL4WDX/PkhnzjlVl0EjNlq5bbvv6dWu3aYrFZqNG3KrUuXEhUfb3RoohIc2bAhP2kCuLKzMVssfs81VRVZuyogiVMpdZVSaodSKlkpNaGI121KqVm+11cppZoUeO0x3/4dSqmyz5orgkJ8ixbcvWkTTzqdPLBnD/W6nreBUlQR1erX99u2RkVx4dix2KpXxxodjTU6Gnt8PFc8+6xBEQZOuW/VlVJm4G2gP3AQWKOU+kZrva1AsbHASa11C6XUKOBFYKRS6gJgFNAWqAcsVUq10lpL03UVcGLnTubedBNpu3ZRs00brvvsM+KaNTM6LFFBrp0xg2l9+6KUQnu9NOzZk4v+9jc63noru+bPR5lMtBoypEqsXVXufpxKqYuB/9NaD/BtPwagtf53gTKLfGV+UUpZgCNAIjChYNmC5c51TZnkI/i5HA5eb9qUrOPHQWuUyURM3brcv3s3FpvN6PBEBck8epRDq1cTWaMGjXr1CrnuRyXtxxmI76o+cKDA9kHfviLLaK3dwGkgoYTHihB0bMsW3Dk5+Y1E2uslNz2dEzt2GByZqEgxtWuTNGQIjS+5JOSSZmkE4jsrqmfz2dXY4sqU5Ni8Eyg1Tim1Vim19vjx46UMUVQ2W2wsHpfLb5/X5cIWYqseClGUQCTOg0DDAtsNgJTiyvhu1asDaSU8FgCt9WStdVetddfExMQAhC0qUkJSEklDhuTPBG+NjqbtqFHUaNzY4MiCn8fj5emnv6dDh3e5/PJPWL/+sNEhibMEoh/nGqClUqopcIi8xp6zV+b6BhgD/AJcDyzXWmul1DfAZ0qp18hrHGoJrA5ATMJgSimu+/xzNn/+Oam//06tdu1oO3Kk0WGFhL//fREffvhb/hwJl146lY0b76J5c+nWFSzKnTi11m6l1HhgEWAGpmittyqlngHWaq2/AT4CpiulksmraY7yHbtVKTUb2Aa4gXulRT30eFwuTu7ZQ2SNGsTUrp2/X5lMdLj5ZgMjC01Tp27wm1jG6fTw5ZfbefjhngZGJQoKyMghrfV8YP5Z+/5V4OscYEQxxz4PPB+IOETlS9u9m48vu4zc06fxuFx0u/deBrz6qtFhhbSzZz8ymRRWa9VtaAlF8tMQ5TL7+uvJ9E3q4MnNZd3777Nz3jyjwwppEyb0yp+D02xWREdHMHJkO4OjEgXJWHVRLie2b/eb7cadk8PRTZtodfXVBkYV2h55pBf168cyd+42atWK5oknLqVOnRijwxIFSOIU5VK9USNO7NyZv22JjCS+RQsDIwp9SilGj+7A6NEdjA5FFENu1UW5XD9rFpFxcXnjke12Wl19NRdcf73RYQlRoaTGKcqlTqdOPLBnD0c2biQqLo5a7dvLbO+iypPEKcotskYNmlx2WZmP114vRzZuxOVwUKdTJyJ8neaFCFZyqw4sWpTMgAHTGTDgUxYv3m10OGHF43IxfcAApl5yCZ8NGsSbLVtycu9eo8MSBtizbBmv1q3Ls1YrH3TrRvrBg0aHVKywT5yLF+/m2mtnsXjxHhYv3s2wYTMleVaite+9x4Gff8aVlUVuejpZR4/y9e23l+jY/T/+yNIJE1j5yivknDpVwZGKinRq/35mDh1K5pEjeN1uDv/2G9MHBO/0vGF/q/7yyytxONz529nZbl59dSVXXtncwKjCx/EtW3BnZ+dva6/Xr5W+OJumT2feXXflzTIeEcHqt97iro0biZRJRELSwV9+8ZtNSXs8pO3aRc7p00H5Mw37GmdRQnCp+ZBVr1s3rHZ7/rbJYqFOx47nPW7RQw/lrWWjNZ7cXLKOHWPzjBkVGaqoQFEJCYX/8JQK2ufdYZ84H374YqKi/qx4R0VZeOihiw2MKLxceMcdJA0diiUyEmt0NDWaNmXo1KkAHPjlF1a++iqbZszIXyXxjIILgEHelHW5GRmVFnc4ObhqFV+MGMHM4cPZs3RphVyjWd++NOjZE2t0NOaICKx2OwNefRWTJThviss9A7wRAj0D/MKFybz66kpA8fDDFzNggHTgrmzpBw/icjiIa9YMk8XCusmTWfS3v+F1uzFZrdTt3JkxK1ZgMpsBmHPjjez46qu8yZIBi93O2J9/lhU1A+zQ6tV8cvnl+f+oLHY7I2bPptXgwQG/1pmlhDMOHaJ+jx406tUr4Nc4n5LOAC+JUwQdrTUv2O35SREgIiaGaz/7jKQhQ4C8FRTn3XUXu777DltsLIPfeYcWV8ma94H2xciRbJs9229fg4svZuzKlaU+17GtW8k6dozaHTpgT0gIVIgBVdLEGZz1YBHWPE4nHqfTb5/WmuwTJ/K3rVFRDP/kk8oOLeyc/YikuH3norVm3l13sfnTTzFZrWivl9ELF9KwZ+hOkxf2zzhF8LHYbNTp1AlV8PmW1jTq3du4oMJUt3vu8Wu8s9rtXPTgg6U6x54lS9g8YwYuh4Pc06dxZmQwO8SH5UriFEHppu++o0GPHpisVmLq1OGG//5XJg8xQLO+fRkxZw4Ne/akXvfuDPngA9rfdPYCD+eWlpzsN4MWkNdf0xO6c5bLrboISjF16nDHTz8ZHYYAWg4cSMuBA8t8fO0OHQrNXxDXtGl+Q18okhqnEKJCNerdm94TJmC22YiIicGemMior782OqxykVZ1IUS5aa8XrfU5a5GOEydwpKYS17Qp5oiIYsvlpqfz7V//yh8//khM3boM+eAD6l54YUWEXUhJW9WlximEKDOtNUsefZTnIiN5zmbjixtuwJ2bW2RZe0ICNZOSzpk0AWYNH872L78kIyWFw+vW8fFll5GRUuSq4YaRxCmEKLMNU6ey5q238LpcaI+HnfPmsfSRR8p8PndODvt++AFPweSrNXtXrAhAtIEjiVMIUWa7FizwG/7qzs4medGiMp/PZLUWORF2wS5RwUASpxCizKo3bIjJav1zh1JUq1evzOczmc30fuyx/ERpttmIbdiwXK36FUEah4QQZeZITeX9Cy8k+9Qp8DUOjf3lFxIvuKDM59Ras23OHPYuW0aNJk3oPn48ETGVs8qnjFUXQc/lcLD4H//gj59+Ir55cwa+8QaxDRoYHZYopdz0dHZ8+y0ep5MWAwaUq8ZpNEmcIqhprZnevz8Hfv4Zd04OymwmulYtxu/Yga1aNaPDE2FKuiOJoJadlsYf//tf/gxI2uPBmZnJH//7n8GRhTbt9frNKiUqhiROYQh3dnaRY5VVCA/DM9r6Dz/khehoXoiJ4f3Onck8csTokKqsciVOpVS8UmqJUmqX73NcEWU6KaV+UUptVUptUkqNLPDax0qpvUqpDb4PmYU2DGSfPMnUSy/122eyWqlWt265lhkOZwd//ZWFDzyAOycH7fFwdPNmZl17rdFhVVnlrXFOAJZprVsCy3zbZ3MAt2qt2wJXAZOUUjUKvP4PrXUn38eGcsYjQsBPEyeScegQukCNMzoxkb+sWoUlMtLAyELXgZUr/ebJ1G43KWvWGBhR1VbexDkUODOb7CfAsLMLaK13aq13+b5OAY4BieW8rghh6QcOFJqo2BIVRWSNGsUcIc4npk4d//6U+BZAExWivImzttb6MIDvc61zFVZKdQcigIILlz/vu4X/j1LKdo5jxyml1iql1h4/frycYQsjNb/ySqwFVi+0REbSrF8/AyMKfW1vuIF6XboQERODNToaq93OMJkhv8KctzuSUmopUKeIl/4JfKK1rlGg7EmtdaHnnL7X6gLfA2O01r8W2HeEvGQ6GdittX7mfEFLd6TQprVm6YQJ/Praa2itaTloENfPmoU1Ksro0EKa1+Nh1/z5ZJ84QcNevUho2dLokEJOpfTjVErtAPporQ+fSYxa66QiysWSlzT/rbX+ophz9QEe1lpffb7rSuKsGrweD9rrxXzWLaYQRqmsfpzfAGN8X48BCs1OqpSKAL4Epp2dNH3JFpU3qn8YsKWc8YgQYjKbJWmKkFTexDkR6K+U2gX0922jlOqqlPrQV+YG4FLgtiK6Hc1QSm0GNgM1gefKGY8QQlQ4GXIphBA+MuRSCCEqiCROIYQoJUmcosrKPHKEz4YMYVLjxswYOJD0Q4eMDilk5WZkcGLnTr/Z3sOZrKsuqiSPy8XUSy7h1L59eN1u0g8dYkqvXozfvl2GdZbSlpkz+fqOO/JWsFSKkV9+SbO+fY0Oy1BS4xRV0okdO8g8ciR//Lb2eMhOS+Po5s0GRxZa0g8d4us77sCdnY0zMxNnRgazhg8P+5qnJE5RJVkiIwtNW6e9XhmdVEqp27cXXs5Xa07/8YcxAQUJSZyiSopr3pymffvmL/pliYqiYc+eJLZta3BkoSWuadNCE7K4cnLAFN6pI7y/e1FlKaUY9eWX9P33v+l4661c8fzz3PTdd0UuPSuKF9esGVc8+6zfzEva42Fq795khfFkO9IBXogQpr1e0nbvBq2Jb9ECr8eD1+UK+DrkE+PiyD11Kn/bbLPR78UXueiBBwJ6HaOVtAO8tKoLUQbu3Ny8iYKVon63boWfA1YCl8PB9P79ObJhA1prImvUwOGrBTa4+GJu/PZbIqtXB/JmpDq0ejVZx45Rr0uXUq9Eqc9+Xuzx4MnNDcw3EoIkcQpRStlpaXzUsycZKSkAxNavzx0rVxIVV+SMihVm+ZNPcnj9+vzF2TKzs/NfO7RqFd/85S/c8MUXaK357+jR7Pj6a0xmM16Phxu//Zaml19e4mu1v/lmNk6bhtvXmm622Ui65prAfkMhRJ5xClFKSx59lFN79+LMyMCZkcHJPXtYNqGoVWMqVsqaNcWuaOlxOvNXDE1esIAdX3+NKyuL3PR0XFlZzLnhhlJda9Cbb9L93nuJb9GCet27c8uSJdRs3brc30OokhqnEKWUum2bX0uzx+nk+LZtlR5H7Q4dOLR6dbG3zGdux0/t24f2ev1ec5w4gdfjyevUXgImi4X+L71E/5deKl/QVYTUOIXw8Xo87F2xgu1ffUXm0aPFlqvfo4ff6CNLZCT1L7ronOc+tX8/8/76V2aPGMG2OXMCEm/fF16gZlJS/nIZZpsNa0wMEdWqYYuN5ZqPPgKgbpcu/r0JlCKhZcsSJ01RmLSqC0HeEM3p/ftzeN06lMmE1poxy5dTr2vhBlaXw8GMQYM4tGoVkJdIb16woNjO9emHDvFuu3bkpqfndcK32+n34ot0Hz++3HF73W6ObNwIWpPQujX7li/HmZVFkz59qFa3bn65XydNYumjj6LMZuwJCdy6fLksrVGESlk6wyiSOEWgrf/oIxbef7/fUMKE1q0Z//vvRZbXvtEzSiliGzY8Z//Qn158kRVPPonX5crfF127Ng8fOeJ/Tq8XVYEdy10OB9knT1Ktbt0KvU4ok/k4hSiF0/v3Fxp/nXGO2ZSUUtRo3JjqjRqdt1O9x+ks9IyxYBI9tW8f77Zvz7NWKy8nJrJ78eIyfAfnZ7Xbia1fX5JmAMg7KKqc1W++yWsNGvBq3br8+NxzlOSuqn6PHn5LFiuLhXpduuB1u1n8j3/wSp06TGrShM2ffQbA5s8+4+XERF6Ijmb29deTdfw4h9asIXXHjkLXu+D66/2eiVrtdi684w4gr+Y6rV8/jm/bhvZ6caSmMmv4cE7t2xeAd0JUFGlVF1XKpk8/ZemECfm1x58mTsRWvTo97rvvnMe1GjyYix96iJ/+/W+UyUR8ixZc9/nnrPjXv1j7zjv55/v2zjvJOnaMZf/8Z36fxp3ffsvOefOw2Gx4XC5aDxvGtZ9+ml+zS2zThluXLmXxww+Tc+oUbUeO5JLHHwfy+oSmHzjgVyM1WSwcWrOGGk2aBPrtEQEizzhFlTJj0CCSFyzw21eve3fu9DXknI/L4cCZlYW9Zk2UUkxq2pTTZ9X+6nTuzJHffoNi/nas0dEMmTyZ9jfddN7reZxOXqhWDW+B7k3W6GhumjePJn36lChmETjyjFOEJVv16nDWM8czww5Lwmq3E52YmP/c0hYT4/e6MpuJrFEDi81W7DlcWVkcL6ZR6WzmiAiumjQJq92OJTKSiJgYmvXrR+PLLitxzKLySeIUVUqfp54iIiYG5Zut3BodTd8XXijz+a589VUsvm5GJouFyOrVGfTWW8TUq4clKgqTxZKXqAska2t0NLVKMX1dt7vvZsyKFfR/5RWGf/opI//7X5nFKcjJrbqock7u2cOGTz5Bezy0v/lmEtu0Kdf5Dq1Zw+9z52KNjqbz2LFUq1eP3IwMNs+YQc7p0yRecAHz7roLZ0YGXpeLtiNHMnTq1DIlP601rqwsrNHRkjwNIP04hahE7pwcUrdvx1a9OnFNm5bpHEc2buSzQYPIOnYMq93OiDlzaN6/f4AjFeciiVOIEOJxOnmtfn0cqan5+6zR0dyfnExMnTrlPr/2etk0Ywapv/9OrfbtaTdqlNRoiyDzcQoRQtIPHsRVYFo4yHumenTTpnInTq01c0aNYtf8+fmPAfYsWcLQKVPKdd5wJo1DQgQBe82a+StynuFxOks94XBRUn//nV3ffYcrKwvIa/Xf8vnnnNq/v9znDleSOIUIArbYWK58+WWsdnv+bEed//IXarVrV+5z56an57X+F2CyWslNTy/3ucNVuW7VlVLxwCygCbAPuEFrfbKIch7gzILWf2itr/HtbwrMBOKB9cAtWmvn2ccLEQ6633cfjS65hCMbNxLfvDmNevcOyHlrtW+PJTKS3IwM0BplMmGrXp2EVq0Ccv5wVN4a5wRgmda6JbDMt12UbK11J99Hwfn2XwT+4zv+JDC2nPEIEdLqdOpEpzFjApY0ASKio7n9f/+jbufO2GJjqdetG7f/+OM5O/GLcytXq7pSagfQR2t9WClVF/hea51URLlMrXXMWfsUcByoo7V2K6UuBv5Paz3gfNeVVnUhREWorCGXtbXWhwF8n2sVUy5SKbVWKfWrUmqYb18CcEprfeaJ+EGgfnEXUkqN851j7fEwXs9ZCGG88z7jVEotBYrqD/HPUlynkdY6RSnVDFiulNoMFPVkutjqr9Z6MjAZ8mqcpbi2EAGnvV7SkpMBiG/RQua4DDPnTZxa637FvaaUOqqUqlvgVv1YMedI8X3eo5T6HrgQmAvUUEpZfLXOBkBKGb4HISqVMyuL6f37c3TjRiCv8eXWpUuJOGtCEFF1lfff5DfAGN/XY4Cvzy6glIpTStl8X9cEegHbdN7D1RXA9ec6Xohgs+KJJzjy22+4HA5cDgdHNm5kmW9+zZJwZWeTc+pUBUYoKlp5E+dEoL9SahfQ37eNUqqrUupDX5k2wFql1EbyEuVErfWZtVQfBf6ulEom75nnR+WMR4gKl7Jund965p6cHFLO01jp9XjYNGMGU3r35t8xMbxcqxYf9ewpCTRElasfp9b6BNC3iP1rgb/4vl4JtC/m+D1A9/LEIERlO3s9c7PNRu0OHYotr71ePrv6avatWJF/jPZ6ObxuHd/eeScjvviiUuIWgSNPtIUopYLrmUfExJDQsiX9Xnyx2PL7f/yRA6Ylhc0AAAVQSURBVD/9lJ80z/A4nRxYubKiwxUVQCb5EKKUbLGxjFu3Ln8989odO2K2Wostn33yZLGt7rENG1ZUmKICSeIUogxMvlUwS6JBjx6FlgdGKWyxsVzzkTzWD0Vyqy5EBatWrx6jFy2ieuPGmG02EpKSuPr997lv165SLbEhgofUOIWoBA179uRBWSu9ypAapxBClJIkTiGEKCVJnEIIUUqSOIUQopQkcQohRClJ4hRCiFKSxCmEEKUkiVMIIUpJEqcQQpSSJE4hhCilcq1yaRSl1HFgfxkPrwmkBjCcQAvm+II5Ngju+II5NpD4zmistU48X6GQTJzloZRaW5LlP40SzPEFc2wQ3PEFc2wg8ZWW3KoLIUQpSeIUQohSCsfEOdnoAM4jmOML5tgguOML5thA4iuVsHvGKYQQ5RWONU4hhCgXSZxCCFFKVT5xKqVGKKW2KqW8SqliuzMopa5SSu1QSiUrpSZUYnzxSqklSqldvs9xxZTzKKU2+D6+qeCYzvleKKVsSqlZvtdXKaWaVGQ8pYztNqXU8QLv1V8qKzbf9acopY4ppbYU87pSSr3hi3+TUqpzEMXWRyl1usB7969KjK2hUmqFUup339/rA0WUMey9K0RrXaU/gDZAEvA90LWYMmZgN9AMiAA2AhdUUnwvARN8X08AXiymXGYlxXPe9wK4B3jP9/UoYFYQxXYb8JaBv2+XAp2BLcW8PghYACjgImBVEMXWB5hn0PtWF+js+7oasLOIn61h793ZH1W+xqm1/l1rveM8xboDyVrrPVprJzATGFrx0YHvOp/4vv4EGFZJ1y1OSd6LgjHPAfoqpVSQxGYorfWPQNo5igwFpuk8vwI1lFJ1gyQ2w2itD2ut1/u+zgB+B+qfVcyw9+5sVT5xllB94ECB7YMU/qFVlNpa68OQ98sD1CqmXKRSaq1S6lelVEUm15K8F/lltNZu4DSQUIExlSY2gOt8t3JzlFINKyGu0jDyd60kLlZKbVRKLVBKGbJ2se/Rz4XAqrNeCpr3rkosD6yUWgrUKeKlf2qtvy7JKYrYF7B+WueKrxSnaaS1TlFKNQOWK6U2a613ByZCPyV5Lyr0/TqHklz3W+BzrXWuUuou8mrGV1R4ZCVn1HtXEuvJG6udqZQaBHwFtKzMAJRSMcBc4EGtdfrZLxdxiCHvXZVInFrrfuU8xUGgYM2kAZBSznPmO1d8SqmjSqm6WuvDvtuOY8WcI8X3eY9S6nvy/iNXROIsyXtxpsxBpZQFqE7l3AKeNzat9YkCmx8AL1ZCXKVRob9r5VEwUWmt5yul3lFK1dRaV8rkH0opK3lJc4bW+r9FFAma905u1fOsAVoqpZoqpSLIa/Co0JbrAr75//btX6WBIIjj+Hcqbf1TiKWVDyAi0SdIERCsTZEmhU9hY5fOLtYWdhaChUkrVuKhFmotFhaWYnEpdgJHQowL3t4Vvw8sWe4uZBiOuexsArR93gamviGb2ZKZLfh8FdgFnkqK5y+5KMZ8AAxy796XbG5sEz2vFqFXVieXwKHvEO8AX+NWTdXMbG3cqzazbUJ9+Pz9Xf/22QacAc95nvdmXFaf3FW1K5VqAPuEJ9U38AFc+/F14KpwXZOwk/dGWOKnim8FuAFe/HXZj28BfZ83gIywi5wBnZJjmsoFcAy0fL4IXACvwB2wkTBf82I7AR49V0NgM/H9dg68Az9+33WALtD18wacevwZM37pUVFsR4Xc3QKNhLHtEZbdD8C9j2Zdcjc59JdLEZFIWqqLiERS4RQRiaTCKSISSYVTRCSSCqeISCQVThGRSCqcIiKRRqEh9G3ntFKpAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# make up a dataset\n",
    "\n",
    "from sklearn.datasets import make_moons, make_blobs\n",
    "X, y = make_moons(n_samples=100, noise=0.1)\n",
    "\n",
    "y = y*2 - 1 # make y be -1 or 1\n",
    "# visualize in 2D\n",
    "plt.figure(figsize=(5,5))\n",
    "plt.scatter(X[:,0], X[:,1], c=y, s=20, cmap='jet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MLP of [Layer of [ReLUNeuron(2), ReLUNeuron(2), ReLUNeuron(2), ReLUNeuron(2), ReLUNeuron(2), ReLUNeuron(2), ReLUNeuron(2), ReLUNeuron(2), ReLUNeuron(2), ReLUNeuron(2), ReLUNeuron(2), ReLUNeuron(2), ReLUNeuron(2), ReLUNeuron(2), ReLUNeuron(2), ReLUNeuron(2)], Layer of [ReLUNeuron(16), ReLUNeuron(16), ReLUNeuron(16), ReLUNeuron(16), ReLUNeuron(16), ReLUNeuron(16), ReLUNeuron(16), ReLUNeuron(16), ReLUNeuron(16), ReLUNeuron(16), ReLUNeuron(16), ReLUNeuron(16), ReLUNeuron(16), ReLUNeuron(16), ReLUNeuron(16), ReLUNeuron(16)], Layer of [LinearNeuron(16)]]\n",
      "number of parameters 337\n"
     ]
    }
   ],
   "source": [
    "# initialize a model \n",
    "#model = MLP(2, [12, 10, 1]) # 2-layer neural network\n",
    "model = MLP(2, [16, 16, 1]) # 2-layer neural network\n",
    "print(model)\n",
    "print(\"number of parameters\", len(model.parameters()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Value(data=1.1560272899423576, grad=0) 0.5\n"
     ]
    }
   ],
   "source": [
    "# loss function\n",
    "def loss(batch_size=None):\n",
    "    \n",
    "    # inline DataLoader :)\n",
    "    if batch_size is None:\n",
    "        Xb, yb = X, y\n",
    "    else:\n",
    "        ri = np.random.permutation(X.shape[0])[:batch_size]\n",
    "        Xb, yb = X[ri], y[ri]\n",
    "    inputs = [list(map(Value, xrow)) for xrow in Xb]\n",
    "    \n",
    "    # forward the model to get scores\n",
    "    scores = list(map(model, inputs))\n",
    "    \n",
    "    # svm \"max-margin\" loss\n",
    "    losses = [(Value(yi) * scorei + 1).relu() for yi, scorei in zip(yb, scores)]\n",
    "    data_loss = sum(losses) * (1.0 / len(losses))\n",
    "    # L2 regularization\n",
    "    alpha = 1e-4\n",
    "    reg_loss = alpha * sum((p*p for p in model.parameters()))\n",
    "    total_loss = data_loss + reg_loss\n",
    "    \n",
    "    # also get accuracy\n",
    "    accuracy = [yi == (int(scorei.data < 0)*2-1) for yi, scorei in zip(yb, scores)]\n",
    "    return total_loss, sum(accuracy) / len(accuracy)\n",
    "\n",
    "total_loss, acc = loss()\n",
    "print(total_loss, acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 0 loss 1.1560272899423576, accuracy 50.0%\n",
      "step 1 loss 1.5468491167071179, accuracy 50.0%\n",
      "step 2 loss 1.2021105302261552, accuracy 50.0%\n",
      "step 3 loss 0.966935857742637, accuracy 59.0%\n",
      "step 4 loss 0.8232331805728894, accuracy 68.0%\n",
      "step 5 loss 0.7356809163892809, accuracy 70.0%\n",
      "step 6 loss 0.6860408935788103, accuracy 74.0%\n",
      "step 7 loss 0.6658413011177632, accuracy 71.0%\n",
      "step 8 loss 0.638051851012845, accuracy 74.0%\n",
      "step 9 loss 0.6089352022220705, accuracy 75.0%\n",
      "step 10 loss 0.5834899120867706, accuracy 77.0%\n",
      "step 11 loss 0.5624132478316041, accuracy 76.0%\n",
      "step 12 loss 0.546382386918789, accuracy 77.0%\n",
      "step 13 loss 0.5251134632393725, accuracy 78.0%\n",
      "step 14 loss 0.5096683473198049, accuracy 79.0%\n",
      "step 15 loss 0.49403581008508085, accuracy 78.0%\n",
      "step 16 loss 0.4809749834485958, accuracy 78.0%\n",
      "step 17 loss 0.45498541176808344, accuracy 79.0%\n",
      "step 18 loss 0.43383641366567494, accuracy 80.0%\n",
      "step 19 loss 0.4133872374672309, accuracy 83.0%\n",
      "step 20 loss 0.3949011845842829, accuracy 83.0%\n",
      "step 21 loss 0.37236244960709985, accuracy 84.0%\n",
      "step 22 loss 0.3584525764878379, accuracy 87.0%\n",
      "step 23 loss 0.36006597889539355, accuracy 88.0%\n",
      "step 24 loss 0.35434445029337647, accuracy 88.0%\n",
      "step 25 loss 0.34973064538700127, accuracy 88.0%\n",
      "step 26 loss 0.3219293076769822, accuracy 89.0%\n",
      "step 27 loss 0.3033779473036923, accuracy 90.0%\n",
      "step 28 loss 0.3075883206062423, accuracy 89.0%\n",
      "step 29 loss 0.29633273929922294, accuracy 90.0%\n",
      "step 30 loss 0.30291923256106246, accuracy 91.0%\n",
      "step 31 loss 0.29390015262943864, accuracy 91.0%\n",
      "step 32 loss 0.3144313347870216, accuracy 89.0%\n",
      "step 33 loss 0.29212341310825113, accuracy 91.0%\n",
      "step 34 loss 0.28619907252632, accuracy 90.0%\n",
      "step 35 loss 0.26494755867649195, accuracy 91.0%\n",
      "step 36 loss 0.2659788754086913, accuracy 92.0%\n",
      "step 37 loss 0.24709652176444358, accuracy 91.0%\n",
      "step 38 loss 0.24009659644742473, accuracy 93.0%\n",
      "step 39 loss 0.23158899730028218, accuracy 92.0%\n",
      "step 40 loss 0.2258913895289618, accuracy 93.0%\n",
      "step 41 loss 0.22122577673795693, accuracy 93.0%\n",
      "step 42 loss 0.21472887164841387, accuracy 93.0%\n",
      "step 43 loss 0.21117262198553868, accuracy 93.0%\n",
      "step 44 loss 0.20507614341297095, accuracy 93.0%\n",
      "step 45 loss 0.20013953460861322, accuracy 93.0%\n",
      "step 46 loss 0.1964456358177856, accuracy 93.0%\n",
      "step 47 loss 0.19240143786907493, accuracy 93.0%\n",
      "step 48 loss 0.18823947603571226, accuracy 93.0%\n",
      "step 49 loss 0.18400235803771584, accuracy 93.0%\n",
      "step 50 loss 0.1797626038261932, accuracy 93.0%\n",
      "step 51 loss 0.17561741189956742, accuracy 93.0%\n",
      "step 52 loss 0.17228466093197506, accuracy 93.0%\n",
      "step 53 loss 0.17081279883838707, accuracy 94.0%\n",
      "step 54 loss 0.16818392663432402, accuracy 93.0%\n",
      "step 55 loss 0.16466978952190214, accuracy 94.0%\n",
      "step 56 loss 0.16166524463144005, accuracy 94.0%\n",
      "step 57 loss 0.1595062524131897, accuracy 94.0%\n",
      "step 58 loss 0.15714423432909805, accuracy 93.0%\n",
      "step 59 loss 0.15442921763772202, accuracy 94.0%\n",
      "step 60 loss 0.15182876804226905, accuracy 94.0%\n",
      "step 61 loss 0.14965676410693154, accuracy 94.0%\n",
      "step 62 loss 0.1469887434341472, accuracy 94.0%\n",
      "step 63 loss 0.1451790450154682, accuracy 94.0%\n",
      "step 64 loss 0.1427369950641003, accuracy 95.0%\n",
      "step 65 loss 0.14136904951614374, accuracy 95.0%\n",
      "step 66 loss 0.13922557320277337, accuracy 95.0%\n",
      "step 67 loss 0.13731961231676967, accuracy 95.0%\n",
      "step 68 loss 0.13552431648854038, accuracy 96.0%\n",
      "step 69 loss 0.13375932497935988, accuracy 96.0%\n",
      "step 70 loss 0.13179063698447666, accuracy 96.0%\n",
      "step 71 loss 0.1330093863236696, accuracy 96.0%\n",
      "step 72 loss 0.12887636059799065, accuracy 96.0%\n",
      "step 73 loss 0.1276828677982079, accuracy 96.0%\n",
      "step 74 loss 0.12551960755933275, accuracy 96.0%\n",
      "step 75 loss 0.1233259329796015, accuracy 96.0%\n",
      "step 76 loss 0.12135168975258047, accuracy 96.0%\n",
      "step 77 loss 0.11973573798933937, accuracy 96.0%\n",
      "step 78 loss 0.12039187195388286, accuracy 96.0%\n",
      "step 79 loss 0.11649522454235622, accuracy 96.0%\n",
      "step 80 loss 0.11689435356248477, accuracy 96.0%\n",
      "step 81 loss 0.11484199341665793, accuracy 96.0%\n",
      "step 82 loss 0.112925220613484, accuracy 96.0%\n",
      "step 83 loss 0.11254571184568385, accuracy 96.0%\n",
      "step 84 loss 0.11058841578362856, accuracy 96.0%\n",
      "step 85 loss 0.11022586042513463, accuracy 96.0%\n",
      "step 86 loss 0.10829250822674177, accuracy 96.0%\n",
      "step 87 loss 0.10783495703193346, accuracy 96.0%\n",
      "step 88 loss 0.10641179072241873, accuracy 97.0%\n",
      "step 89 loss 0.10649242657616906, accuracy 98.0%\n",
      "step 90 loss 0.10361360829750184, accuracy 97.0%\n",
      "step 91 loss 0.10274561962901459, accuracy 97.0%\n",
      "step 92 loss 0.10196629664724012, accuracy 97.0%\n",
      "step 93 loss 0.10513600083138182, accuracy 98.0%\n",
      "step 94 loss 0.10349532329503071, accuracy 97.0%\n",
      "step 95 loss 0.1078303551517095, accuracy 99.0%\n",
      "step 96 loss 0.0983792616810677, accuracy 97.0%\n",
      "step 97 loss 0.09902510890529767, accuracy 98.0%\n",
      "step 98 loss 0.09487484218844873, accuracy 97.0%\n",
      "step 99 loss 0.09395068058424742, accuracy 98.0%\n",
      "step 100 loss 0.09320413108627204, accuracy 98.0%\n",
      "step 101 loss 0.09145462351112094, accuracy 97.0%\n",
      "step 102 loss 0.09045449606655677, accuracy 98.0%\n",
      "step 103 loss 0.08922940874060949, accuracy 97.0%\n",
      "step 104 loss 0.09018392335032248, accuracy 98.0%\n",
      "step 105 loss 0.09066785615901775, accuracy 97.0%\n",
      "step 106 loss 0.09538308825512637, accuracy 99.0%\n",
      "step 107 loss 0.08726797374328281, accuracy 97.0%\n",
      "step 108 loss 0.09063783962723128, accuracy 98.0%\n",
      "step 109 loss 0.08597910317867449, accuracy 97.0%\n",
      "step 110 loss 0.08697285707651278, accuracy 98.0%\n",
      "step 111 loss 0.08617423007080864, accuracy 97.0%\n",
      "step 112 loss 0.08693833506351614, accuracy 98.0%\n",
      "step 113 loss 0.08522733944329712, accuracy 97.0%\n",
      "step 114 loss 0.08500750952567426, accuracy 98.0%\n",
      "step 115 loss 0.08361050211737113, accuracy 98.0%\n",
      "step 116 loss 0.08438468331122048, accuracy 98.0%\n",
      "step 117 loss 0.08288908389973328, accuracy 98.0%\n",
      "step 118 loss 0.08345229818791665, accuracy 97.0%\n",
      "step 119 loss 0.08281701326801028, accuracy 98.0%\n",
      "step 120 loss 0.08249248870718336, accuracy 98.0%\n",
      "step 121 loss 0.08250123860896062, accuracy 98.0%\n",
      "step 122 loss 0.08157829333252911, accuracy 98.0%\n",
      "step 123 loss 0.08189386809115651, accuracy 98.0%\n",
      "step 124 loss 0.08068217549166593, accuracy 98.0%\n",
      "step 125 loss 0.08157710844894994, accuracy 99.0%\n",
      "step 126 loss 0.07989165795172712, accuracy 98.0%\n",
      "step 127 loss 0.07969578610177709, accuracy 98.0%\n",
      "step 128 loss 0.08013155583016364, accuracy 98.0%\n",
      "step 129 loss 0.0793773105457011, accuracy 98.0%\n",
      "step 130 loss 0.07916645782778285, accuracy 98.0%\n",
      "step 131 loss 0.07905841250712287, accuracy 99.0%\n",
      "step 132 loss 0.07822185440869583, accuracy 98.0%\n",
      "step 133 loss 0.07889204201149531, accuracy 99.0%\n",
      "step 134 loss 0.07746976873858831, accuracy 98.0%\n",
      "step 135 loss 0.07706834739699926, accuracy 98.0%\n",
      "step 136 loss 0.07677825120803373, accuracy 98.0%\n",
      "step 137 loss 0.07646331394292918, accuracy 98.0%\n",
      "step 138 loss 0.07628556701316, accuracy 99.0%\n",
      "step 139 loss 0.07594080996149011, accuracy 99.0%\n",
      "step 140 loss 0.0763461402329126, accuracy 98.0%\n",
      "step 141 loss 0.07745914407485807, accuracy 100.0%\n",
      "step 142 loss 0.07551594862792824, accuracy 98.0%\n",
      "step 143 loss 0.07530607223283833, accuracy 99.0%\n",
      "step 144 loss 0.0745777423477899, accuracy 98.0%\n",
      "step 145 loss 0.07514006846930242, accuracy 99.0%\n",
      "step 146 loss 0.07380363739259455, accuracy 99.0%\n",
      "step 147 loss 0.07453186204781374, accuracy 99.0%\n",
      "step 148 loss 0.07309419710868192, accuracy 99.0%\n",
      "step 149 loss 0.07271855573427059, accuracy 99.0%\n",
      "step 150 loss 0.07253154370164279, accuracy 99.0%\n",
      "step 151 loss 0.07207487968886142, accuracy 99.0%\n",
      "step 152 loss 0.07167442934935335, accuracy 99.0%\n",
      "step 153 loss 0.07151086127100373, accuracy 99.0%\n",
      "step 154 loss 0.07107723137313131, accuracy 99.0%\n",
      "step 155 loss 0.0715122945539185, accuracy 99.0%\n",
      "step 156 loss 0.07082684964416852, accuracy 100.0%\n",
      "step 157 loss 0.0706050001290405, accuracy 99.0%\n",
      "step 158 loss 0.07016037651870584, accuracy 100.0%\n",
      "step 159 loss 0.06981785249805375, accuracy 99.0%\n",
      "step 160 loss 0.070221367352654, accuracy 100.0%\n",
      "step 161 loss 0.07006568635440873, accuracy 99.0%\n",
      "step 162 loss 0.07117395928811533, accuracy 100.0%\n",
      "step 163 loss 0.06856617565480125, accuracy 99.0%\n",
      "step 164 loss 0.06809252155511225, accuracy 100.0%\n",
      "step 165 loss 0.0691880041117283, accuracy 99.0%\n",
      "step 166 loss 0.07300888514118122, accuracy 100.0%\n",
      "step 167 loss 0.06723301271332696, accuracy 99.0%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 168 loss 0.06889133150887823, accuracy 100.0%\n",
      "step 169 loss 0.06585882693226622, accuracy 100.0%\n",
      "step 170 loss 0.065534224126955, accuracy 100.0%\n",
      "step 171 loss 0.06642120780859764, accuracy 99.0%\n",
      "step 172 loss 0.07036344441064256, accuracy 100.0%\n",
      "step 173 loss 0.06434457178446804, accuracy 100.0%\n",
      "step 174 loss 0.07057410384496458, accuracy 100.0%\n",
      "step 175 loss 0.06360567978062148, accuracy 100.0%\n",
      "step 176 loss 0.0683835917621888, accuracy 100.0%\n",
      "step 177 loss 0.06095292627441029, accuracy 100.0%\n",
      "step 178 loss 0.06251457342696964, accuracy 100.0%\n",
      "step 179 loss 0.06011189328891595, accuracy 100.0%\n",
      "step 180 loss 0.06270048859585112, accuracy 100.0%\n",
      "step 181 loss 0.05766597058490163, accuracy 100.0%\n",
      "step 182 loss 0.06672680890554591, accuracy 100.0%\n",
      "step 183 loss 0.06178847289409725, accuracy 100.0%\n",
      "step 184 loss 0.05778653286930187, accuracy 100.0%\n",
      "step 185 loss 0.05451700258994074, accuracy 100.0%\n",
      "step 186 loss 0.05435893176864256, accuracy 100.0%\n",
      "step 187 loss 0.053020730846958734, accuracy 100.0%\n",
      "step 188 loss 0.051820820361933286, accuracy 100.0%\n",
      "step 189 loss 0.05156730473001523, accuracy 100.0%\n",
      "step 190 loss 0.050839390731608564, accuracy 100.0%\n",
      "step 191 loss 0.050607696868470775, accuracy 100.0%\n",
      "step 192 loss 0.05002918583215331, accuracy 100.0%\n",
      "step 193 loss 0.04983677282429972, accuracy 100.0%\n",
      "step 194 loss 0.049342555028681956, accuracy 100.0%\n",
      "step 195 loss 0.05184882295082756, accuracy 100.0%\n",
      "step 196 loss 0.05189844827072028, accuracy 100.0%\n",
      "step 197 loss 0.05678443434786125, accuracy 100.0%\n",
      "step 198 loss 0.0558898500599847, accuracy 99.0%\n",
      "step 199 loss 0.0480839875543577, accuracy 100.0%\n"
     ]
    }
   ],
   "source": [
    "# optimization\n",
    "learning_rate = 0.001\n",
    "\n",
    "for k in range(200):\n",
    "    \n",
    "    # forward\n",
    "    total_loss, acc = loss()\n",
    "    \n",
    "    # backward\n",
    "    model.zero_grad()\n",
    "    total_loss.grad = 1\n",
    "    total_loss.backward()\n",
    "    \n",
    "    # update (sgd)\n",
    "    for p in model.parameters():\n",
    "        p.data -= learning_rate * p.grad\n",
    "    \n",
    "    if k % 1 == 0:\n",
    "        print(f\"step {k} loss {total_loss.data}, accuracy {acc*100}%\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-1.548639298268643, 1.951360701731357)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XuMZPd14PfvufdW9fv9fs50z5NDipT5ph6WZEkWJWgleVcGpASBN9kFkWAVJ0H+WBlGdoMFAjhwkGB3bcShbcHeYG2vQUc2d1cL2pS8lmSbNCnxTc5w3jM93T39ftfz3pM/blV1V1dVP6u7urvOByA4XVVd9051zzn3/n7nd36iqhhjjKk+TqVPwBhjTGVYAjDGmCplCcAYY6qUJQBjjKlSlgCMMaZKWQIwxpgqZQnAGGOqlCUAY4ypUpYAjDGmSnmVPoGttNXUaX9Dc6VPwxhjdiwSSQDg1kSROge8KNTUhE86EeJ+HHCYTwThQ+KxvKhEHLcsx5+5e2VGVbt28tojnQD6G5r5g89/s9KnYYwxOzLUewuA+tEhIg/XA+A8/SkApK6bywtXAXjhhgBQ67byo5fCRNBTV56L3d/+5U/e3ulrbQjIGGPK4CgE/9060ncAxhhzHGwO/k73MIyOABuDfz0v3IgBRyP4gyUAY4zZl70G/0oG/ixLAMYYs0fZ4N/ytQsAecH/SmIREotsDP6vvdwMHI3gDzYHYIwxe7Jt8AcKg39lh3w2swRgjDG7dBKCP1gCMMaYXTkpwR9sDsAYY3ZsqPdWfpnnMQ7+YHcAxhizI6WCv9R1FwT/Wrf1yAd/sDsAY4zZ0u4WeMWOTI3/TtgdgDHGlHAcV/fuhiUAY4wp4qQHf7AEYIwxBaoh+IMlAGOMyVMtwR8sARhjTIH60aHcn53uYSAM/hteAYTBP+u4BX+wBGCMMVXLEoAxxlQpSwDGGFOlLAEYY0yVsgRgjDFVyhKAMcZUKUsAxhhTpSwBGGNMlSpLAhCR74jIlIi8W+L5T4vIooi8mfnvn5XjuMYYY/auXO2gfw/4DeDfbPGaH6nql8t0PGOMMftUljsAVf0hMFeO9zLGGHM4DnMO4BkReUtE/pOIPFjqRSLynIi8LiKvzydih3h6xhhTXQ4rAfwUOKWqjwD/GvjTUi9U1edV9XFVfbytpu6QTs8YY6rPoSQAVV1S1ZXMn78HRESk8zCObYwxprhDSQAi0isikvnzk5njzh7GsY0xZi+yewFslN0LYKPsXgDHUVmqgETkD4FPA50iMgb8cyACoKq/BXwd+O9EJA3EgG+oqpbj2MYYUy5DvbfyN4LpHobRkbyNYKCeF24cr83fSylLAlDVb27z/G8QlokaY8yRs9NdwE5S8IfyrQMwxphjaXdbQJ6c4A/WCsIYU8Wqaf/fYiwBGGOqUrUHf7AEYIypQhb8QzYHYIypKpuDf7bSB6or+IMlAGNMFdlZ8A8rfWA9+J+0wJ9lCcAYUxWywb/laxcA8oL/lcQiJBbZGPxfe7kZOLnBH2wOwBhTBbYN/kBh8D95Qz6bWQIwxpxoFvxLswRgjDmxLPhvzRKAMeZEsuC/PZsENsacONXW1G2vLAEYY06Mam3qtleWAIwxJ0I1N3XbK0sA5siILyeYvTlHbCmBOFDfUkfbcCu1TTWVPjVzxFlrh72xBGCOhNnb89z5yT3UX98naPHeMhOXp+i92E3/gz0VPDtzlFnw3zurAjIV56f8guCfE8DUlWlWZ9cO/8TMkWfBf3/sDsBUVGwpzsR799Gg9A6hga9MXZ9lpKNwj9bNVJW1uRgaKPXtdTiuXeOcVNbUbf8sAZiKuff2BFPXZotf+W+ycHeRxIM91DRES75mZWaV6z++hZ8OQEFE6H+kl55zneU8bXMEWFO38rAEYCpiZXqV6R0GfwANlHtvTTD6sVNFn08l0lz9q5t5dxKqyr03J4jWerQNte7oOMm1JHffnGBpYhmAlv4mBj/aT7QusqPvNwfPmrqVjyUAUxEzN+cIdhj8sxYzQbno+12fLTmMdOeNcRAhUuvR0FGPiBR9XTrpc/nla6QTfu6xhbElVmbWePDZ87gRd1fna8rPVveWlyUAUxF+Ktj195SI2wCszJSeJPYTPrdfGwPAq/E497Mj1DQWDiXN3JwLh48KztVn9tY83TaUVFEW/MuvLDNkIvIdEZkSkXdLPC8i8q9E5JqIvC0ij5bjuOb4ah1sxnELI7o4IE6RSC/QOthS8v1qm7deKxCkA4J0QHI1ydUf3kRVSawmufnKHd787nu89WfvM1NiSEp9ZXlqdfu/lDkwFvwPRrlKJH4PeHaL578InMv89xzwf5fpuOaYahtqoba5Ni/YiyPUtdRx6skBxBXIPOW4DtG6CAOP9JV8v75L3Ts+djqRZnFimcsvX2P+7iJBOsBP+iTXUsW/QSDaYHMAlWLB/+CUZQhIVX8oIqe3eMlXgX+jqgq8IiKtItKnqhPlOL45GhIrCRbuLaGB0tLfTF1LbcnXOo7D+c+MMn19ltlbCwB0nGql62wHjuvQ0N7AzI05UmspmrobaBtu3bKk04t6nHpykNt/N7ajc529OUdQZLinGHGErjMdaKAsToRzApFaj/ZTbURqbRT1IFlTt4N1WL+9A8DdDV+PZR4rSAAi8hzhXQJ99U2HcnJm/yYvTzHx3hSgqMLEB1N0jrQz+NG+kpOujuvQc76LnvNdBc/VNEQZ+Ejvrs6h41Qbzd2NzNyaI7mWYmlihVSs8KpeAyW+ktxy7YHjZZKNwvATA3g1Hh/8xVWSaymCdIA4wvg7k3i1Hn4qoLYpSv9DvTT32u9sOVhfn8NxWAmgWAQo+q9PVZ8Hngd4sL1nd2UipiLWFmJMvD+VX4LpK7M352jua6Jli6CoGr5u4oNpUrEUNQ1R+h7soX14Z2Wbm0XqIvQ9ELaNWLq/wvW/vpU3ri+u0DbYQjrhk1hKFHy/OELvxS7q2+sAoam7Acd1uPOTeySWk4Q3seT+rqlYOvwM5uNc/5vbnH5icMclp6Y4W917eA4rAYwBQxu+HgTGD+nYVSexmmTqwxlW59aoaYzSc76L+ra6Azve7M35opOnga/M3JjbMgFMXp7m/gdTuZLQxEqSO6+PEaQDOkfb93VezT2NnPn4ae69NUFsKY4Xdek620HvA90sT62wPL1SeN4CHSPtROvzx/zn7izkgn8p6itjb03QOthS8q7HbM2C/+E6rATwIvAtEfkj4Clg0cb/D8baQowP//IGgR+uhl2bi7Fwb4lTTwzSXqYrU1VlcWKZqQ9nSCfSW46l+0m/5HNBOsgL/rnHfeXeO5N0jLTtO5A29zTS/PPnCh5v7GqgY6SN2Rvz6xPRqpx+ergg+ANbDhdtlE74pJM+kRqbG9irgjF/Cnv527BPeZTlt1RE/hD4NNApImPAPwciAKr6W8D3gC8B14A14L8ux3FNoTuv3ysIyOord39yj9aBZhxn/4Vf4+9OMn11dtuFXI4rtA6W/ocZX0lkivuL3T0EpOLpA1mBO3NzjntvTaAaJrNI1KPnQhftp9twveKfT1NPY2518HZKvYfZnWzwz1fYD8qC/96Vqwrom9s8r8A/KcexTGlBOmBtIVb0OVVYm4/R2NGQe8xP+cSXE0RqPaL1pXvsbJRcSzL1YelVtxt5dRE6TpcexvFqvNLvoxzIytvFyWXG3hjPS17JtRSTl6e3HHIafKSPKzOruT5DxYgjYZK1BnTmmLD71JNkm9GS7NW/qjL+ziRTV2cRR9BAaeioZ/SZYbxthi6W768gUjIG5p3LmY+d2vJqOFoXoaGznpXp1bw3FCe8cziIK+nJ9wuHnCBMhouTy7T2F7+arG2q4YGfP8f9KzMsT6/gRV0Sy8lMQgjfr7alluHHBsp+zsYcFEsAJ4jjOjR1N7I8tVIQoV3Poa41rMufvjobNmILNHcFvjK9yuUfXKe2qQb1A9qGW2kvUnsvnlNy2CbvdSLbJhOA0aeHufqjm2FFjgiqSkN7/YEF0sRKsujjGgQFz6WTPosTS6ivNPc2Eq2PMvQz/evfo8rK9CqJ1SR1LbXUt9XZ5K85ViwBnDCnHh/g8vevE6R8Al8RVxARRp4ZzgWnycvTRa+CkytJkpkguDq7xsz1Oc5/ZjQvCbT0Nm1bDQNQ11q7o0VSXo3HA587x9p8jMRKktrmmi0XkO1XbXMNK9PpgsfFcajb0E5i7u4Ct18bCz8zDdc2dJ/vzFubICI0dTdilf/muLIEcMJE66M8+MULzN2ZZ20uTk1TlI4NK1ZVlXSiMABuFvhKbCnO7K15us505B53Iy4jTw9z85U7oIXVMY7r4HjC6aeGNr/llurb6g60VDWr78Eerv3oZn75p0Ck1qOppxEI5zluvzaG+opuuNOZvjpDY2cDLX3lC/nZfZCTsRTNPU20DbXYHII5NJYATiDXc+ga7YDRwudEhGh9pHTfmw3UV+ZuL+QlAD8dkIqnwsVUyTQ1DVEauxpJJ9IklhPUttTSNtR6ZCthmroaOP3kEGNvjJPOlKg2dTdw6omh3B3S7O2F3Lj+RoGvTF2dKVsCyO2DHCgoLI4vM/nBFBc+exYvaq2nzcGzBFCF+j/SE5aL7qAf/8ZmbYnVJFe+fy3srOkr4ggrzhrtp9po26JT51HTNthC60AzqXga13MKqo3S8TRaYmnDTu6ediKdLNwHOUiH8xAT704y9KhNJpuDdzQv08yBah9uY/Bn+vFq3DDAC0UriBxX6Djdlvv69mtjpBN+LnFooATpgJt/e2dH8wJHiYgQrYsULTVt6mlc7wW08XscKVuvn6XJLTa3uTVflmMYsx27A6hSnSPtdJxuI530cT2H8XfvM3N9fXGX4zk0dtbnevL4aZ/VmeI98dOJNPGlxIFO3h6mlr4maptqiC3G1+c4BNyIU7ZNYVS15BoI9RU/HRzZYTRzclgCqEKxpTipWIq6lloiteFK28FH+mgbbGH2dtjXp3Wwhebexty4eKkhESAs39xhq4TjQEQ49+lRJj+YYu7WPEGgtPY30/dQT9naPzf3NpVeUCawMr1CS5+tcDUHyxJAlUgn0szcnGP66iypZDpT3gjtw60MPzaAOEJDRz0NHYVL7QG8qEttcy2xxXjBcyJyYq7+s1zPYeAjvbtuSb1TkRovHHYrkgTEqoDMIbEEUAUWx5e48cqdvAnHbHnj3N0FIvUR+h/s2fZ9hh8f4Opf3cw1moOwvfKpJwaKb+NogEzyvTHHyswaNU1Rus50UNtUQ9tAC/Nji0W/p6m78ZDP0lQjSwAnnJ/yubkp+G+kvjJ9dYa+S93brmJtaK/ngc+f4/6H06zNxahpitJzoYv61oOv3z+u4ssJrvzgOoEfhD+D+zBzY47RZ4YZ+GgfyzOr+Ck/fE7CiebhxwdsLYA5FJYATriF8aVt+/b4qcwV/Q4u4msaowxbieKO3fnpvfyW2Bom3Vuv3uXhr1ziwWfPM3trnuXpVaL1kdzdgTGHwRLACRZfijP53lTJq/+saH3EhnB2KLGSYPLKDKszq0Qbwjugpq6Goq8NgiBsdFeEKqzOrdHY2UD3uc6yVRcdZ9nNYLJ7AQC5zd/X9wIgt/l7di8As3eWAE6oVDzFlR9cD6/utyCu0H9AE50nzebNduJLCVamVhh8pI/ODauld0JVGXtzgnQyTV1rHX0PdB9KK4yjKhv8W752AVjf/B3gSiI7T1KfC/6vvRxWSNleAPtjCeAEScVSTN+YI7YYx9+wYKsUr9Zj8JG+Pe+/e9iyQaIc7k6e3v33/HS8YLOdILMNZNupws1kHMehqauB5anCuwD1lbX5MJglV1MsTS5z5uOnae6pvsnfod5bhbuAjY4U7AL2wg3bBazcLAGcEKtza1z9q5t5LZ5LiTZEOPvJkWM11rxxr9iyvB+3dpUENFBWZ9eKPynC6uxa0eA99NgAV76/YRK4ROmn+srdn97j0rPnq6aldNH9fy34HypLACeAqnLzlbtb7s2bJQ60DbUe2+CfNz68D/UMMUT4vjtKBBIu0CrV8cJxiwft2sYaHvzieWZuzLM6u4rjuSzcWyw6L5NcS+En/R3to3Dc7W7zdwv+B+Xk/6ZVgeRqklR8++6eEC7a2mrrw6Om5FXiPgRTd4g8XE89Q6zduMtQ7/Z3AyJCy0ALC/cWC67gHVdKLqAD8KIevRe7gC7iy4nwPYrQQBl/d5K+B3tyK7RPot0Ffyz4HyArNj4BtivzdDwHx3Nwoy6jHz9FTcPO9v+ttM2Bwuke3nfwh/UEEnm4PjektJP5haFH+4nWR3ON4sQVHM9h9GOndjxsU9MY3XL/5Zmb83zw51dJxcvTdfSoseB/tNgdwAlQ0xAlUuMV7fEfbYxy+olBRIT69uOzZWGx4J+tCikHZ3SE4JW/2tWdQKTG49Kz51i8t8TqXIxofYT2U6140Z3/MxIRRj82zNW/vIGfnRfYSMPFe1NXphl4pG+Pf7ujyYL/0WMJ4AQQCXfguvbDWwRBWKIoTrjN4chTQzS0l2fc/LBsVRIodd1lOYbGpnCe/tSuk4DjOLQNtdI2tPfKqbrmWh768kVuvnKHxfHCttAawMLE0olKAFsldAv+lVOWBCAizwL/EnCB31HVX9v0/D8Efh24l3noN1T1d8pxbBNq7GzggS+cY/raLLHFOPVtdXSd6SBaf7zGkretB08UHz/frYut5/acBMrBcR0auxpZmlwpWrVVbJ+C42pnwX+9xj8b/A8i8GsiTfrOIjoXg4iLO9SM01l/bO6My23fCUBEXOA3gc8DY8BrIvKiqr6/6aX/TlW/td/jmdJqGqIMHuOrxp0uBirGTys/fjHBT7+fJBGDnmGHz36jluGLxX/FLy9czUsC3LhJ5OE7h5oE2oZaGH9nsuBxxxW6zu5uYdlRtbOEvnmB1wEF/3ia5N/dg3S2mWGK9GICZ7CZyLnjUxhRTuW4A3gSuKaqNwBE5I+ArwKbE4A5ItKJNGNvTbAwtogqNPc2MvhIPzWNlZsc3u1K0M3G/zjN2jVFM3Onk7cC/uD/WGPgv3SpO1VY6/D10frcsMOFmhYYHcEBIg/foeXhCyz+6ZUDSwJ+ymf29jxrczFa+pvCYSAJq4DEEVoHW2gfbiUIAuZuLTB7O9whrON0Gx2n2o5N247DWN2rKR9/bIlgPo7UeriDzTjNxUuc09fnYfPK+EAJxpbQoWakTHs9HCfl+BsPAHc3fD0GPFXkdf9ARH4W+BD4n1T1bpHXmAMW+AGXv38tnDDOjDwsji+zMn2NS8+eq0j5YamVoFA8UNS6+ePv8ek0a9fmc8E/S9Mw+wPh7D/e9Hp/gRduxPj6aD2wxpXEYl4SCKbu0PK1g0kC2X2V/XQ4AZwN5l1nOojUejT3NlHXUosGytW/uklsPpZb0b02H2Pu9gLnfnbkyCeBQwn+sRTJ18bBVwjCBufB/VXcCx14/YVbdwYzJRbyAcFcDLfI95x05UgAxX4TNw9q/nvgD1U1ISL/LfD7wM8VfTOR54DnAPrqq+8HctDm7y6STvgFP6HAD5i6OntgG6CUsts2AJBpAqaKu5zGjftI3KcmKF7TvHYvXdA07JNfaK1YErj9erivclZ2/H/uzgIf+fLF3Fj0/Ngisfl4XjsP9ZW1uTUWxpdoG2wpy/kchMPq65O+Olf0it6/Movb3YBs3lKzVNKULZ474cqxDmAM2Lg+fxAY3/gCVZ1V1UTmy98GHiv1Zqr6vKo+rqqPt9VUb3Osg7IyvVp0xbAGysr0yqGdx1DvrcLg//SnCoL/Czckrw3Aj14K6HYbaHt7maYPlqm/sUbdeBynxCJoibj01DXn/fejl4Jc0AmDUHj8XHAaHcmtFcgGsXL0IQr80t1Bg3RAbGF9t7X5u4th07mC91Dm7y7s+1wOymE2dSt5RS8QzBfuXOf0NRaPeApO5/GqlCuXciSA14BzIjIiIlHgG8CLG18gIhtnJr8CfFCG4554saU4t/7uLh/8+VVuvnIn1zxsP7Zq/RzZYoFSOe23Hjz17hS6lsrd+rNVB4x6D93Yj5/1YJO9o9iYBHJlphuSwG4Wi21FlZKr9gI/yKsGKtVaAsJS1KMom9A3B3+p6y4I/rVu6/47em5VuVPkKW+kFWmMQvazzVz5ew92Ft4tVIl9DwGpalpEvgW8RFgG+h1VfU9E/gXwuqq+CPyyiHwFSANzwD/c73FPuuWpFa7/+BZBoKAQW4yzML7EyNPDtPbvvUKiY6SNySvTBY+LK3SfO/jKk/0Gf0366EK89PLnzc3W5hMkX71H9KkBJFpYWlnrhsNBeW9R143GpvIeqx8Nq4P2w/UcxJXi+zMo1G6YvOw43cbi+FJBR1fHdegYadvXeZTbTn+m5W7q5nTVE9wvfkfltBXuUS2uQ+TxfoLZWDhpHHVwexuRKui9VEpZ/uaq+j3ge5se+2cb/vwrwK+U41jVQFW5/fq9gn/86it3Xh+j+csXWRxfZvr6LH7Sp3Wgma6znXhFAtxm0fooo88Mc/OV9WCmqgw+3EdjR/GNTcqlHCtBNeWH47XbtLrOk/JJ31ogcr6ypZV+OijZqVVcIbYYp7Ez/Bk09TTSNtzK/J2F3O+B4wrtp1ppLLEBTSVUsqmbd66d5EI8nAcIMp1WRfAudSIlttQUEdzOetwqHfLZrHpT3xGWjqdJxYo3dwt85eard1meWM4FhvhSgpkb8zzw+bM76iTZ0tfMw195gOWpVTRQmrobDnzhUbnaAEjdFlVKTol2nQrB9BpUOAFsOWKReXL21jzJtSR1rXUMPdpP50g782PhHUrbYOuWTecO22G0dtBA0fkYGihOay2y4fdUajyizwziT6yg83GodfEGmpFjtvixkiwBHEGyxfivBsrivaW8YQ4NlHQ8zf0r0ww8vLOFYI7r0NJ3OFVWe10JCoWBQhzBPduOf3UuvOrLcgTpqken14omgaNQNpn9zBcnlguGsFSVD//yRu5rcYVofZQLnxll8JH+Qz7T7R1G8A9mY6Teub/+gIJ7tg1vaL0CSlwHb7AZBq1dxF5U58zHEedFvZLbA2pmTqDgcVXmx8rTJqGcNlaF7LYNQKlA4Q02417qhDov7NPfEMF7qItIqTkMR3AGjkZJ8fBjA0TrInkdRYGCeQH1lcRygrtvjBe8R6UdypV/Ik3q7fvhUF/2v0Dxr80TlKEYwoTsDuCIOv3kEFd+cI0grWE5YImdpDY6atUhB9UGIFiI41+fh2w9vSNIfQSpcXEvduBfng3vAhRwBWmpwT0iV4iR2ggPfvECC+NLxBZipOLpcKVviUqmhbElVPXI9Ko5rKZu/uRK8eG8QElfmyfyeO2R+UyOM0sAR1RNY5SHvnSR+bsLrC3Embk+u2X8F1foOEIbvRxUPbiupUi9MZk3/KPLSVKvTxD92CBeXxNuWx3+/RU0HeC21yGtRytYiCO0DbbQNtjC7Z/c27KMVTXc4nOrYcHDcphN3TSeLnnBo0sJ0h/M4D3QmfdzVVWCqTX8e0uQDnC66nEHm/PmDUw+SwBHmOM5dIy00+YHTF+fLfk6cYSG9nq6zh6NBHCQi4HSdxbzx/6zAsWfWMEbbkFqPbxTx2Oj+2hduC6jVHVQXWstTomKlsN02E3dnNZagomVktVewf1VtK8J2VDumb48QzC5mvv98FdT+PeWwxJgSwJFWQI4BsQRonWRohu+IHD6qUFaB1pKXuUmVpNMfjDNyvQKkVqP7vNdtA4czJDIQa8E1ZVk8ScCLf3cEdYx0sb9y1NFL3bFgeFHBw79nDY7zNW9WU5XA1yfh1iJndECxb+/kqv3D1aSecE/+xqSPunbi0SOyMXRUVP5SwuzLRFh4JHegmEAxxU6z3TQNthaMvjHl+J88OdXmb01R2IlycrMGrdevcP4e/eLvn4/Sq0EhfIFCmkoUeLnSOnnjrBoXYSRZ4ZxMovEsitY61prufi5cxUv+6xE8IfMRc8T/RDdWYgKZotXf6EQTBVfLGbsDuDYaBtsRRyHe29PkFhJ4kU9ei500n2+c8vvG3t7sqD3T+Ar9y9P5zpQlsNum7rtdTGQO9xSeKUHIBzbbo7r6zJWMusyGne0LiO+nAgXigVKS38TDe3hxiaqysLYEnO350Gg/VQbrQPNu54HqVTwz5KIi3e2nfSV2cKhIEdwexryvi75PkegBPiosgRwjLT2N++qDUR8OcHSZOGWgxD+o1ieXqF9H1sbwuGvBHUaokQe7iH1/nS4sQeEY/4PdRUd59VEmmAxgXgO0na0JoM3CtcI7PzzmLw8xcR7U2im2mn66gwt/c2cenKQG399O2z6lwmay/dXmOtuZPTjO9+8vtLBP8vpaUQmVtClxHoScAWnuwFpXR//d7sa8K/NF94FOIJzTC8MDoMlgBPq3ruTTF2Z2bJ01N1nA6xKbfLtdNQR/cQQupYKA1qdVxDYVJX0tTmCsaX1JbiOEPlob8kNQ46L2FKcifen8iaOA19ZHF/m3tuTrEyv5bURCXxleXqVxYnlHV1AHNbd3E5I5mcWzKwRTK6EV/59jUh7Xd7PXGo93HOZBYIbS4CbwxLgYCUJiTTSGK3q3j+b2SdxDK0txFiaWM7tHlXTkN/Fc2VmlakPZ0pWlgAg0NTduOdzOMzgr6roXAx/fBl8xelpwOlpxGko3b00mFwlGFvOlFhmPgdfSb0xSfQTQyW/7ziYu7NQ9Gcb+AFztxeKt5FOB8zdWdgyAVSqqZtqZgI/6SONNUhN/p2cOILb3YDbvXUPJG+wGbe9Dn8yUwLcUQ8NHqnXx9HVVG4tjdPTgHex04aGsARwrKgqd396j9nbYQAQgfF379P/UA89F7pyr5u5MVe842SG4wpnPnZqz+WFh33ln74yG5YEZoJeMB9HxpaIPNpXsumXf2eheLmoathH/misC9sTze1pW+zJ0t+31ehPpZq6aTxN6s1JNJbOBGjF6WvCu9Cxp+E6qY/gja53S02+eq+gOiy4v4pf4+KdscogqwI6RhYnlpm7vRgGdwUNwtYQ4+/dJ7a4vgGGn/JLvkddSy0Pffnivq7+Yb1HPpDrm5/rpR++AsjfvnFPm34sJfKCf/igoiuSdEjPAAAdcElEQVSp8I6ghM17AOR9b2YF8cadwrJj2etXueTGvLMBEcqzMcx+tfQ3F03e4gotA01F9xJwXIf2U8XbSFdqKE9VSb4xEV6dB9l2DxBMrODf2X9bk2A5Ge4bUfCE4o+V/t2pJpYAjpGZ67NFb+/V17DiI6N1oKVkEOi+0IkXPT43fumxpZILv4LJ0uV9TnNhP3gARHBaanKB60cvBRs2hhEg3Cw+m8ycpz+F0z1M5OH6sm0Ms1+NXQ00dTfk/Yyza0UGH+mjua8pL0E4rtDS30RzT2HSr1Twh3BFL4kiiTpQ/DtL+3pvCAsAim5YC5AOwgn0KmcJ4BjxN+9/WuK5tqEWahpr8sY4xRFqmqJHei/ZjVSV1JUZdGKP21Q2FZkfEJDmGiQzCbxxi8jNu4NtTALZ3cEiD9eXdYvIvRIRRj9+iqFHB2joqKeutZa+B3u4+LmzeFGPkaeHGf3YMO2nWmk/1crox09x+qmhgiGVSgZ/AI2XvlNli7vYnXIao8XXBgDUFhYOVKPjcyloaB1sZm0hVjC+73gOLRtK3RzX4fzPnWH66gyzt8Ne8h2nWuk+13kk2grsRDAbIxjfIvg7Eu7xWux7l5MEt4sMISi459oL/uGHSWAJaOaJzy3lNovPBsDD2Cx+t0SEjtNtdJwuHNYREZp7m2jubSKdSIe7ym0QBAHdzbfwfaHp3PC+W3TvldMULTlnUY6e/lLr4XQ3hgvBNrUO984erV3VKsUSwDHSOdrOzPU5kmupXBWIuEJ9Wx3Nvfm1zq7n0PtAN70PdBd7qyMvuFdi6AcyV/LRkgu//LHFkld+wcQybpEy0J66Zu7Hlnjt5fwkAGtcSSweySSwlcRKklt/dze3j3Sk1mPosQFWZ9eYvjqFanhXePEX5njsMxcOrKnbVqQ+gtNRRzAbKwjQbplaN3gPdOLXuvh3l8I5hloP72wbbpHhsGpkCeAYcT2Xi587y9TVGebuLOI4QsdIG51n1q9qVZWliWWmb8zhp3zaBlvoGGnD9Y5XMyxNlx7uktZaIh/tLVnGpwm/dDXMFsMOJyUJBOmAKz+4RnrD+HpyLcX1H9/CcSDIPaxc/rNpgo5bPPXrowfS1G073kPdpK/PEdwLS3yp8/DOdZRty0ZxBO9MO+5oG6itCt7seIwHmBw34tJ3qYcHnz3PAz9/LhzW2bAPwN03xrn5yh2WJpZZnVnj3juTXP6La1tWBh1FTldD8eX9ruAONW/5D3nLfkHtJSaHM7KBLruyNTsnABtWwGbmBIAjMSew2fzYIkG6RF+cTb8Gftznw++8w/sz05lHDn5170biCJFzHUQ/dYroZ05T87Eh3K56NOkTLCXCPaDLcRwRC/5FWAI4QdYWYszems9bBaq+klxLMXV1poJntntufxPUuPlJwBGkMYqzxdVhsBAnuFuigiTi4O5gG8y9JIH60aEjkwRii/Gi1WIleUJibJXDDv6Q2c70ziKpV++RfGWM1IezJN++T/Kv75D66QTJH98l9f701osazZ5ZAjhBliaWi/5D0UCZK0Nd9WESzyH65ADuqRak3kMaIrhn2sLFX1tUb6SvzJYc/ok81ofssP3FxiRQ67YWJAGp685LAtky0aOQBGqbaoqWAZcSpAKifR2HH/xVSb05iX99PlwLEEsT3F0K93UOyG0DGdxfJf1h6f0wzN6VJQGIyLMickVEronIt4s8XyMi/y7z/KsicrocxzX5xJGSqz2P4+2veA7eaBvRZ4aIPj0Ybvayxd9Dt9kTIP3mfYLlxI6Pv3mtwAs3YrmSyFyZ6OhIrnRyYxKoZCJoG24pvkLaAWdTe2Wpden42jn+dC7MmocV/AF0Lh6uBdju6j5QgokVdDd3NWZH9p0ARMQFfhP4InAJ+KaIXNr0sn8EzKvqWeD/Av73/R7XFGodaKHYyhdxhc6RKih7E0ov/CGzneRPJtBYkdWhJZReMEbBgjHgSCwYcz2X858ZpandwY0IXq1DTYvLo//9JXo/OYRT4+I2hXsod3z5DG/98jPUuq2HGvwB/Nm1kjt+FVVqdbfZs3LcATwJXFPVG6qaBP4I+Oqm13wV+P3Mn18APiu2CqPsahqj9D3Uk7dxjOM61LfV0VkNfU9SQd4WgUUFSnqXq0yPWxIY6r3F+fOTfPl/uchXf+8hPvGrI3jNDbz9u1eZfm0SBdq/MsL4f/gmb337U9TWdx5oR89SxHO2TNgFoserku04KEcCGADubvh6LPNY0deoahpYBDrKcGyzSe+FLi5+9izd5zvpGGlj5Okhzn96NK9S6KTRdEAqM3Goi9sM8Sjohr5JO3VcksDm1b11HRFe+VfjrN5dIr2SIrWUQBM+9//kBjWv3Dvwds5bcXsbt+5Ql+VkKr+OySLG46Qcn2ixn+Dm+7qdvCZ8ochzIvK6iLw+n4jt++SqUV1LLYOP9HHq8UFa+ne/E9Rxk35vKuzwmZ043IbU7W2V6VFPAsVaO4wvDJIusvbBiadp/v13Khb8IVwI5p5rDyu9ssN3jkBjJIxMjoRlv6dacM9UwRBmBZQjAYwBGxusDwLjpV4jIh7QAswVezNVfV5VH1fVx9tq6spweiebBlrVJXIaTxPMxbdsg5zHCQNK7vtVw9YR87EtF59lFe8fVPkmcqX6+qzdW0FL9JDyM602KhH8s7zBZqLPDOKeaw+rvJ7op+apQaKfOk30Y4NEf/YU3mhb0YsYTfnohlXxZvfKsRL4NeCciIwA94BvAP/Fpte8CPwS8LfA14EfqLXi25fEapK7P7nH0lT4j7ipq4GhRweobTq83a42tknOymunnLGx7XK5aSwdXsaUOkTEyasycS905HYE07UUqbfuo/FsL3rCq82R1uIBJ7sxzVycjqjDj/99wCf+XitxfyGvf9DF1nNobCq3ajjy8B1aHj64VcNbNXXrfOYi6vyw8O8CLAz0VDT4Z0mthzeU36RQHIESO3dp0if13jQ6HwuHkBzBPdeOZ1s/7tq+E4CqpkXkW8BLgAt8R1XfE5F/Abyuqi8Cvwv8vyJyjfDK/xv7PW41Syd9rrx8jfSGqojlqVWufP86l754nsgBbnmXDTabtwsECnaMAg58iEHqvdJlhJ5D5BNDsBz2m5eW9Q6pGijJn04UtCP2by9CnRfuNiaEWwiKoH5A6o3JsMzUVxBoEeGVpQZSnZVrIrc5+G9u6jZ74UMaPtLF4htTOBt+X/yIx/2/94WynMNW1A/QhTiq4LTW7ngdRsn3UyX104mwz78S9nwKFP/KLBJxcbvK00KiWshRvhB/sL1H/+Dz36z0aRw59z+cZvzd+wVdQcUReh/oou9Sz4Ec9yjtFbtR6t0pgum1woZio214p4q3v/Zn1ki/O1V6ziBbSeU5RD7STTAbw7+9UHin4QgLj7eC5/DE58Lqoq+P1gFrQCYJANy4STB1B4DFP72yl79mScU2bt/Y1O1P3lui+V++SsN/uArxNKsDvdz6B19i+cypsp7HZv70Kun3ptcfUHAvduDtYDU2gMZSBKupsKtnY9jeO1iIk3pzsujPTRqjRJ/aXH9SfX77lz/5E1V9fCevtWZwx9DKzFrRLR81UFZm1sp+vErtFbtT3qUu0h9mto0EEHBPt+IOb3HseHrreYPs5+v7pN6YBNcpPswk0LniMdMa7KqJXOrt8vycit2JFTR1q4vw8uNfgseUntomOISKMI2lSL87XXB35l+exWmM4mwxVKmBZib2Y7mhOWmIEHmkd/3Kv8Qxze5YXdUxVJsZniggUNtYeqP0vdjNpiGVCP6QaSh2sZPozw4TfSYzcXi6+Dh+7nuKbRhTigKlJogVCIJd9w8qNn+yWyWDP1C0r099y6EEf8js5FZsdCHQsDXzVt97bS4M/rltIhVdTpJ6+35YwVVqtXudXc/uln1ix1DnmQ6mr8/mNX2DMBB2nS3f8opK7xi1W+I64ZX6Tl7bXIM0RcNWBNuNggZhm2Ji6aJPO+1htdp27aQvtp5DR8ndCZQjCew4+B/2zyNeuiW3xtOoHxDcXw2H7jwHd6AJp7U2rMq6t1x0XkdXkmjUQWq9oncC6ivBQhyndZvFgCbHEsAxVNMYZeSZU9x69U7uIksETj0xSG2pvXB36bgF/90SESIf7SV9dTbcWzhQ8ASKtVF2Bae/keDOUngnoBsfb8pbV7AxCXzyCw4v3FjIJYFchdBoGW+9j2Lwh3BF9sxaYSB3BGmpIfXaeFjBlXk+mFoNK7BOtZTexlFAkj6RR/tIvnMfFjYt+oulSb0xSeSxvlyll9maJYBjqqWviYe/conV2TUUaOioK9tq35Me/HPSAU5TDdJUg9NRhwDJV+8VTjC6Tlim2NdE+s4iOhuDiIs73Fy0NXU2CfzopYBPfqGVF24sAMLXRzUvCZTLUQv+EK7y9W8tFG767goE5AV/IBwaur2I09sQtnwotlm8gjREkahL5Ex7ODezOcEEin9jHuejveX/S51AlgCOMXGExq6Gsr7ndmWFJyX4p2/MhyWfGf7VcL/gyKN9pC/P5LqKSmstkQc6c8NLkXMdcG7799+cBMK1AvlJoBwOagJeE+nwM6jNlMTukngO0Sf6SX84RzCzCgpORz3e+XaSxQI3gCrBTAz3bDv+BzMFVV1OfyOS6QcULCVKb/u5i46v1c4SgMnZWfA/+L1iD1rq1gLBzYWCx/2rczhP9hN9ciBcFSzsq//MdkmgXMJkXJ7gr4GSvjxDMLkStmLYUIEjNbtrxiY1HpGPFO5JnSnsKfIN4f+83kZEIH1tPqzWiji4wy15K7ilxgvPr1g5aNTC2k7ZJ2WA9eBfrKa8EnvFHpRgdo3g+nyJJxV/fBnnXMe+FyxllU4C5WlzsjkZZ4+5V/7NeYL74RV7NriGFTiTRJ8oT42909sY3n0V3AVIbiGX29OI29OIqhat5nK66uFKkQTgCO7p4ms/TCErAzXbB3/gKI0v70f6atEWVOuS5W9bUbyJ3P4bHZY7+KtmSjSLVuCkCFZLb7azG+5wS1iyuWm7T/d0S0GjvlKlvOIIkUf7wm1D3bBpHA7hvEx3eYdFTzK7A6hy1RT8gXDrwVIccDoOpgHhehJY2nAnsP8kUNY5mA1X/QWEcDhmD/MBBW/lOUSe6CeY2lQG2rK7CjanMUr040PoUgJNBTgtNUjE9gzYDUsAVazagj8QNocr0R2TGm/fV4+aDsIgGnWKXr2GnUTDJFAO5ZyAF0eg1g1r+DdTRcq4yFBcB7evCXeHbSFKvo8IssvEYdZZAqhSVRn8AXeoJSxP3DzM4QmRJ/r3vHeyxtOk3p9GFzKbzdR4RB7ozC0S2yibBMqh3D8P90yJCpyehnDidZf8mTX82wto3MdpjuKOtOX6+pjKswRQhY5qU7fD4J5uCZuMZatcyFSrfLRnz8MHGijJ18fza9fjaVJv3SfyeF/RvjdH9bP0ehsB8K/PhXcCbrgbl7uHPaXTdxbxr8+vL/aKpwlmYkQe7d31cI85GJYAqkw2+GdtDP7r1oN/1lENWLslIkQudaGjbQQrSSTqIk3Rfe2aFkyvFe8VFCj+zQWchw+mO2u5aTpAF+M4NR7O4/3oagqp93Bqd7+DmqaDvOCfEyipNyaJfmK4bJVWZu8sAZiqJLUebm15fv11NVly8jRY2X3ljPoB/u1F/PFl8BWnow7vTNuet7LcifTdJfxrc4Cudz3N3CE57XV4D3WhC3HS1+fRtRRS4+GOtIb7+hb7OywlSm/47iup96aIPmKrdSvNEoAx+yR1XliGWCQJOPWFQVs17G4JFNx9qIZXyLqcyAXi4P4qydkY0acGkE1JS1XRxQQaSyEN0T31wAnmY2HwL3K1DhDMxcJNWFZSucd0LUX6gxk0nsY7XWRC2936jkpnYmjKt6qdCrMEYMw+Od0NcHWuxKKk/OAYzMdIbdyIxhG8S124mZ5COh8PWzBsHlHyA9K3F4lcWO/2qomw+ZnGM11KNdwUJfIzvYjnhEM6ywmIulu2cyi+KGvjSSu6VOROJjPE5Q41F6yYluaakkkRAFfQVGAJoMIsARizDxoo/sRy2MAs2ynUARwH90J7XmtijadJvXk/P9j6SvqdKYK+BnQhgab84kFTIZhcwe+oCxvXiZB6Z6pgXYMuJ0i9P4XTWBMG9uyGKnUekUd6ig4j5RLIXkjYpnlzKaaI4D3cQ/r1idLfWqYhOLN39hMwZo9UldSbk+hiYj2oC2EJ6BP9OJuubrfaJCW4t7L9AdMB6Xenwk3UH+zKDSPlnxTodAx/Np6XaHQ1RfKnk0Q/Nlgw4S0tNVsvkNuKUvIq3m2pJRhtI7g5n9/8J7Nd515Lbk352DS8MXsUzMTCyc6NV/QKJPywn85mW2xnuGO+huPv1+dLT7JC8SGdlI/Oxwse9k63bj1m7wjUe4XHk7BRnBSZ51h/7xbcCx1hywaAGhf3QjvesPXrOQrsDsCYPQqmV4sP1wRKMLUKg/mls9JcA7Oxrcfbd0JB52LhLkC74Svp63NEWvMXvEldhMhjfaSvzIZ3M7Ae7B3BHW7BGWoON2PfOBcQdYlsU+IqIngDzXgDJ6OM+KSxBGDMXm3RKlqKXFG7A03bT7julIIz2kJwc9P7ORLOR5QY19flJOkb80TOtuc97jTVEH28H904RJUOwAtbWmjSDxe6OaxPUKcC/Jk1vEEL7sfVvoaARKRdRP5CRK5m/l90uaCI+CLyZua/F/dzTGOOCrevMb+jZe4JwSlyxSsRN2w30VoTXmELYe+dPQyFS0sNkdNtuBc7cp01pSGC91AXkQc6S//LVgjGlvID/cb3FVn/L+Lm5gvS1+cg6edXJwWKf3UunLg2x9J+7wC+DXxfVX9NRL6d+fqfFnldTFU/us9jGXOkOM01uKda8q/qM31zSnUVdeojRB/rR/1sxZCQeiszkexrYTIoFqcdwTsfloN6fU1QpKGae6oVv8imN0B4HGVXiSeYWit+LgLBbKzkgjBztO03AXwV+HTmz78P/GeKJwBjTiRvtA2np4Hg/iqqitvVsKPFWBvr5iMf7UUX4vizMcRzcNpqSX84t76aNtulMwCnpQb3dGtBXb+qZtYPKNJUEw433VooHrRrPavAMcD+E0CPqk4AqOqEiBTu/xaqFZHXgTTwa6r6p/s8rjFHhtMQxRnde4dLEUHa6nDa6sLGcn9zd72xXLZHf8wn8lhfQXLxp1dJfzi3PuafSRjeA504fY0Ek6sFcwTeud03dnO66wkmVgoTih7cHgrm4G2bAETkZaBY045f3cVxhlV1XERGgR+IyDuqer3E8Z4DngPoq99fr3BjjptgerV0Y7lb+Y3l/PsrpN+fKSxD9ZX0+zN4j/YitRH8u4uQCpA6D/dcO27X7vc88M60k5yNhXspbBjucs+322reY2zbBKCqnyv1nIjcF5G+zNV/HzBV4j3GM/+/ISL/GfgZoGgCUNXngecBHmzvKUO5hDHHh66mdtRYTlXDTdNLVRQFSnB3ichD3XgjrSX31t0pibpEnx7En1hG5+JhPf9AM06T9fY/zva7EOxF4Jcyf/4l4M82v0BE2kSkJvPnTuDjwPv7PK4xJ5LUR0ouysprLBdoyVLPLF1bX927n+Cfew/PwRtqIfJID5GLnRb8T4D9JoBfAz4vIleBz2e+RkQeF5HfybzmAeB1EXkL+EvCOQBLAMYU4XTVF08AjuCOtOZ9XbQEdQPZQ2dQU132NQmsqrPAZ4s8/jrwjzN//hvgI/s5jjHVQlyH6GP9YaO3tdT6pO6FjrxdtEQEZ6CJ4N5SYedQAFfwTlm7BbM1WwlszBEj9RGiTw2g8TSaDsJ+O0WGcLyz7aRjaYK5tfwk0BAhcqnrQDeQMSeDJQBjjiip9bZcqyWOEHmkB11LhRPEEQdpjBZ0ITWmFEsAxhxzUh/B3aIjpzGlWDtoY4ypUpYAjDGmSlkCMMaYKmUJwBhjqpQlAGOMqVKWAIwxpkpZAjDGmCplCcAYY6qUJQBjjKlSlgCMMaZKWQIwxpgqZQnAGGOqlCUAY4ypUpYAjDGmSlkCMMaYKmUJwBhjqpQlAGOMqVKWAIwxpkpZAjDGmCq1rwQgIr8oIu+JSCAij2/xumdF5IqIXBORb+/nmMYYY8pjv3cA7wJ/H/hhqReIiAv8JvBF4BLwTRG5tM/jGmOM2SdvP9+sqh8AiMhWL3sSuKaqNzKv/SPgq8D7+zm2McaY/TmMOYAB4O6Gr8cyjxljjKmgbe8ARORloLfIU7+qqn+2g2MUuz3QLY73HPAcQF990w7e3hhjzF5smwBU9XP7PMYYMLTh60FgfIvjPQ88D/Bge0/JRGGMMWZ/DmMI6DXgnIiMiEgU+Abw4iEc1xhjzBb2Wwb6CyIyBjwD/EcReSnzeL+IfA9AVdPAt4CXgA+AP1bV9/Z32sYYY/Zrv1VA3wW+W+TxceBLG77+HvC9/RzLGGNMedlKYGOMqVKWAIwxpkpZAjDGmCplCcAYY6qUJQBjjKlSlgCMMaZKWQIwxpgqZQnAGGOqlCUAY4ypUpYAjDGmSlkCMMaYKmUJwBhjqpQlAGOMqVKWAKpU5OH6gscuL1wteOxHLwWHcTrGmAqwBFBFhnpvUT86lAv+TvcwjI4gdd0bgn89L9yIUeu25oJ/T11zhc7YGHOQ9rUfgDkehnpvAeQH/6c/BZAX/F+4IYAFf2Oqhd0BnHC7C/5Y8DemilgCOMEs+BtjtmIJ4ISy4G+M2Y4lgBPIgr8xZidsEviE2Rz8s5U+sDH4h5U+sB78LfAbU30sAZwg2eDf8rULAHnB/0piERKLWPA3xmTtKwGIyC8C/yvwAPCkqr5e4nW3gGXAB9Kq+vh+jmsKbRv8gY3B/7WXmwEL/sZUs/3eAbwL/H3g/9nBaz+jqjP7PJ4pYm/B38b7jal2+0oAqvoBgIiU52zMrlnwN8bs1WFVASnw5yLyExF57pCOeeJZ8DfG7Me2dwAi8jLQW+SpX1XVP9vhcT6uquMi0g38hYhcVtUfljjec8BzAH31TTt8++qzm74+YMHfGFNo2wSgqp/b70FUdTzz/ykR+S7wJFA0Aajq88DzAA+29+h+j33SFK3xt6Zuxpg9OPAhIBFpEJGm7J+BnyecPDZ7ZMHfGFMOorr3i2wR+QXgXwNdwALwpqp+QUT6gd9R1S+JyCjw3cy3eMAfqOr/tsP3nwZWAaseCnVin0WWfRbr7LNYZ58FnFLVrp28cF8J4DCIyOu2biBkn8U6+yzW2Wexzj6L3bFeQMYYU6UsARhjTJU6Dgng+UqfwBFin8U6+yzW2Wexzj6LXTjycwDGGGMOxnG4AzDGGHMAjkUCEJFfF5HLIvK2iHxXRForfU6VIiK/KCLviUggIlVX7SAiz4rIFRG5JiLfrvT5VJKIfEdEpkSkqtfViMiQiPyliHyQ+bfxP1T6nI6LY5EAgL8AHlLVh4EPgV+p8PlUUrYDa9GV1CeZiLjAbwJfBC4B3xSRS5U9q4r6PeDZSp/EEZAG/mdVfQB4GvgnVf57sWPHIgGo6p+rajrz5SvAYCXPp5JU9QNVvVLp86iQJ4FrqnpDVZPAHwFfrfA5VUymn9Zcpc+j0lR1QlV/mvnzMvABMFDZszoejkUC2OS/Af5TpU/CVMQAcHfD12PYP3SzgYicBn4GeLWyZ3I8HJktIXfSdVREfpXwdu/fHua5HbYydWA9iYptPGFlbAYAEWkE/gT4H1V1qdLncxwcmQSwXddREfkl4MvAZ/WE166WowPrCTUGDG34ehAYr9C5mCNERCKEwf/fqur/V+nzOS6OxRCQiDwL/FPgK6q6VunzMRXzGnBOREZEJAp8A3ixwudkKkzCLQl/F/hAVf/PSp/PcXIsEgDwG0AT4WYyb4rIb1X6hCpFRH5BRMaAZ4D/KCIvVfqcDkumEOBbwEuEE31/rKrvVfasKkdE/hD4W+CCiIyJyD+q9DlVyMeB/wr4uUx8eFNEvlTpkzoObCWwMcZUqeNyB2CMMabMLAEYY0yVsgRgjDFVyhKAMcZUKUsAxhhTpSwBGGNMlbIEYIwxVcoSgDHGVKn/HznPVdn9p2kWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# visualize decision boundary\n",
    "\n",
    "h = 0.25\n",
    "x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1\n",
    "y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1\n",
    "xx, yy = np.meshgrid(np.arange(x_min, x_max, h),\n",
    "                     np.arange(y_min, y_max, h))\n",
    "Xmesh = np.c_[xx.ravel(), yy.ravel()]\n",
    "inputs = [list(map(Value, xrow)) for xrow in Xmesh]\n",
    "scores = list(map(model, inputs))\n",
    "Z = np.array([s.data < 0 for s in scores])\n",
    "Z = Z.reshape(xx.shape)\n",
    "\n",
    "fig = plt.figure()\n",
    "plt.contourf(xx, yy, Z, cmap=plt.cm.Spectral, alpha=0.8)\n",
    "plt.scatter(X[:, 0], X[:, 1], c=y, s=40, cmap=plt.cm.Spectral)\n",
    "plt.xlim(xx.min(), xx.max())\n",
    "plt.ylim(yy.min(), yy.max())\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}