badam_weight_pruning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.python.ops import control_flow_ops\n",
    "from tensorflow.python.ops import math_ops\n",
    "from tensorflow.python.ops import state_ops\n",
    "from tensorflow.python.framework import ops\n",
    "from tensorflow.python.training import optimizer\n",
    "import tensorflow as tf\n",
    "#import tflearn\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from optimizers import BayesAdam\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import _pickle as cPickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batches(X,Y, batchsize = 128):\n",
    "    for i in range(0, len(X), batchsize):\n",
    "        yield X[i:i + batchsize], Y[i:i + batchsize]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "mnist = tf.keras.datasets.mnist\n",
    "\n",
    "(X, Y_),(X_test, Y_test_) = mnist.load_data()\n",
    "X, X_test = X / 255.0, X_test / 255.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((60000, 784), (10000, 784))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train = X.reshape(X.shape[0], 28*28)\n",
    "X_test = X_test.reshape(X_test.shape[0], 28*28)\n",
    "\n",
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn.preprocessing\n",
    "label_binarizer = sklearn.preprocessing.LabelBinarizer()\n",
    "label_binarizer.fit(range(max(Y_)+1))\n",
    "Y_train = label_binarizer.transform(Y_)\n",
    "label_binarizer.fit(range(max(Y_test_)+1))\n",
    "Y_test = label_binarizer.transform(Y_test_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((60000, 10), (10000, 10))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_train.shape, Y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "inds = np.random.permutation(X_train.shape[0])\n",
    "# X_val = X_tmp[inds[:5000]]\n",
    "# Y_val = Y_tmp[inds[:5000]]\n",
    "X_train = X_train[inds]\n",
    "Y_train = Y_train[inds]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#X_val.shape, Y_val.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((60000, 784), (60000, 10))"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape, Y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gaussian(x, mu, sigma):\n",
    "    scaling = 1.0 / np.sqrt(2.0 * np.pi * (sigma ** 2))\n",
    "    bell = np.exp(- (x - mu) ** 2 / (2.0 * sigma ** 2))\n",
    "\n",
    "    return scaling * bell\n",
    "\n",
    "def show_weight_dist(mean, variance):\n",
    "    sigma = np.sqrt(variance)\n",
    "    x = np.linspace(mean - 4*sigma, mean + 4*sigma, 100)\n",
    "    plt.plot(x, gaussian(np.array(x), mean, sigma))\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(X, Y, X_test, Y_test, badam = True, num_epochs=10, learning_rate = 0.0001, l2 = 0.01, dropout= 0.0, \n",
    "          beta1=0.9, beta2=0.999, prec=100.0, laplace_mle=False, no_bias_init=False, size=60000, act='relu'):\n",
    "    \n",
    "    # learning_rate = 0.001 ## good for ~ 10-20 epochs\n",
    "    # learning_rate = 0.0001 ## good for ~ 30-100 epochs\n",
    "    \n",
    "    \n",
    "    cv_accuracy_results = []\n",
    "    val_loss = []\n",
    "    \n",
    "    tf.reset_default_graph()\n",
    "    tf.set_random_seed(1)\n",
    "    \n",
    "    inputs = tf.placeholder(tf.float32, [None, 784])\n",
    "    labels = tf.placeholder(tf.int32, [None, 10])\n",
    "    training = tf.placeholder(tf.bool, ())\n",
    "    input_d = 784\n",
    "    \n",
    "    if act =='relu':\n",
    "        act_fnc = tf.nn.relu\n",
    "    elif act == 'sigmoid':\n",
    "        act_fnc = tf.nn.sigmoid\n",
    "    else:\n",
    "        raise ValueError\n",
    "    \n",
    "    with tf.variable_scope('nn'):\n",
    "        regularizer = tf.contrib.layers.l2_regularizer(scale=l2)\n",
    "        _hidden = tf.layers.dense(inputs, 400, \n",
    "                                  kernel_initializer=tf.initializers.random_normal(mean=0.0, stddev=0.1, seed=None),\n",
    "                                  bias_initializer=tf.initializers.random_normal(mean=0.0, stddev=0.1, seed=None),\n",
    "                                  kernel_regularizer=regularizer,\n",
    "                                  bias_regularizer=regularizer,\n",
    "                                  name='dense_l1')\n",
    "        hidden_d = tf.layers.dropout(_hidden, rate=dropout, training=training)\n",
    "        hidden = act_fnc(hidden_d)\n",
    "        \n",
    "        _hidden = tf.layers.dense(hidden, 400, kernel_initializer=tf.initializers.random_normal(mean=0.0, stddev=0.1, seed=None),\n",
    "                                  bias_initializer=tf.initializers.random_normal(mean=0.0, stddev=0.1, seed=None),\n",
    "                                  kernel_regularizer=regularizer,\n",
    "                                  bias_regularizer=regularizer,\n",
    "                                  name='dense_l2')\n",
    "        hidden_d = tf.layers.dropout(_hidden, rate=dropout, training=training)\n",
    "        hidden = act_fnc(hidden_d)\n",
    "        \n",
    "        logits = tf.layers.dense(hidden, 10, kernel_initializer=tf.initializers.random_normal(mean=0.0, stddev=0.1, seed=None),\n",
    "                                 bias_initializer=tf.initializers.random_normal(mean=0.0, stddev=0.1, seed=None),\n",
    "                                 name='dense_out')\n",
    "        out = tf.nn.softmax(logits)\n",
    "        \n",
    "    tmp = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=labels, logits=logits))\n",
    "    loss = tmp + tf.losses.get_regularization_loss()\n",
    "    opt = BayesAdam(learning_rate=learning_rate, N=size,\n",
    "                    laplace_mle=laplace_mle, no_bias_init=no_bias_init,\n",
    "                    params = {'beta_1': beta1, \n",
    "                              'beta_2': beta2,\n",
    "                              'prec': prec})\n",
    "\n",
    "    weights = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=tf.get_variable_scope().name)\n",
    "    \n",
    "    opt_train = opt.minimize(loss)\n",
    " \n",
    "    correct_prediction = tf.equal(tf.argmax(out,1), tf.argmax(labels,1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    \n",
    "    with tf.Session(config=config) as sess:\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "\n",
    "        for epoch in range(num_epochs):\n",
    "\n",
    "            i=0\n",
    "            for x, y in batches(X,Y):\n",
    "                sess.run(opt_train, {inputs: x, labels: y, training: True})\n",
    "                \n",
    "#                 if i <100:\n",
    "#                     print(sess.run(weights[3]))\n",
    "#                     xxx = opt.get_slot(weights[3], \"sigma\")\n",
    "#                     print(\"sigma: {}\".format(sess.run(xxx)))\n",
    "#                     print(\"m: {}\".format(sess.run(opt.get_slot(weights[3], \"m\"))))\n",
    "#                     print(\"v: {}\".format(sess.run(opt.get_slot(weights[3], \"v\"))))\n",
    "                    \n",
    "            acc, l = sess.run([accuracy, loss], {inputs: X_test,  labels: Y_test, training: False})       \n",
    "            cv_accuracy_results.append(acc)\n",
    "            val_loss.append(l)\n",
    "\n",
    "            print('Epoch: {:>2}: Validation loss: {:>3.5f}, Validation accuracy: {:>3.5f}'.format(epoch, l, acc))\n",
    "    \n",
    "        sigmas = [opt.get_slot(v, \"sigma\") for v in weights] \n",
    "           \n",
    "        # store flattened arrays:\n",
    "        if laplace_mle:\n",
    "            mean_coefs = [opt.get_slot(v, \"mean_coef\") for v in weights]\n",
    "            weightvalues = np.hstack(np.array([sess.run(c).flatten() * sess.run(w).flatten() for c, w in zip(mean_coefs, weights)]))\n",
    "        else:\n",
    "            weightvalues = np.hstack(np.array([sess.run(w).flatten() for w in weights]))\n",
    "        \n",
    "        sqrtsigmavalues = np.hstack(np.array([sess.run(tf.sqrt(s)).flatten() for s in sigmas]))\n",
    "        \n",
    "        # store weights:\n",
    "        _weights = [sess.run(w) for w in weights]\n",
    "        \n",
    "        def resetweights():\n",
    "            for v,_v in zip(weights,_weights):\n",
    "                v_t = state_ops.assign(v,tf.cast(_v, v.dtype))\n",
    "                sess.run(v_t)\n",
    "        \n",
    "        def pruning(remove_pct, new = True):\n",
    "            if new: \n",
    "                sorted_STN = np.sort(np.abs(weightvalues)/sqrtsigmavalues)\n",
    "            else: \n",
    "                sorted_STN = np.sort(np.abs(weightvalues))\n",
    "            cuttoff = sorted_STN[int(remove_pct*len(sorted_STN))]\n",
    "            #print(weights[-1])\n",
    "            #print(sess.run(weights[-1]))\n",
    "\n",
    "            for v,s in zip(weights,sigmas):\n",
    "                if new:\n",
    "                    stn = tf.abs(v)/tf.sqrt(s)\n",
    "                    mask = tf.greater_equal(stn,cuttoff)\n",
    "                else:\n",
    "                    mask = tf.greater_equal(tf.abs(v),cuttoff)\n",
    "                #print(sess.run(mask))\n",
    "                v_t = state_ops.assign(v,tf.multiply(v, tf.cast(mask, v.dtype)))\n",
    "                sess.run(v_t)\n",
    "            #print(sess.run(weights[-1]))  \n",
    "            \n",
    "            acc = sess.run(accuracy, {inputs: X_test, labels: Y_test, training:False})\n",
    "            print(\"%.2f, %s\" % (np.sum(sorted_STN<cuttoff)/len(sorted_STN),acc ))\n",
    "            return acc\n",
    "        \n",
    "        xs = np.append(0.1 * np.array(range(10)), [0.95, 0.96, 0.97, 0.98, 0.99, 0.992, 0.995, 0.997, 0.999])\n",
    "        ya = []\n",
    "        print(\"Pruning smallest elements (baseline):\")\n",
    "        for pct in xs:\n",
    "            ya.append(pruning(pct, new= False))\n",
    "        \n",
    "        yb = []\n",
    "        resetweights()\n",
    "        print(\"Pruning using signal-to-noise from Bayes:\")\n",
    "        for pct in xs:\n",
    "            yb.append(pruning(pct, new= True))\n",
    "        return xs, ya, yb, cv_accuracy_results, weightvalues, sqrtsigmavalues, val_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Scale of 0 disables regularizer.\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x13b71a4e0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x13b71a4e0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x13b71a4e0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x13b71a4e0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dropout.call of <tensorflow.python.layers.core.Dropout object at 0x10e144f60>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dropout.call of <tensorflow.python.layers.core.Dropout object at 0x10e144f60>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dropout.call of <tensorflow.python.layers.core.Dropout object at 0x10e144f60>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dropout.call of <tensorflow.python.layers.core.Dropout object at 0x10e144f60>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x13b71de48>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x13b71de48>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x13b71de48>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x13b71de48>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dropout.call of <tensorflow.python.layers.core.Dropout object at 0x10e144f60>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dropout.call of <tensorflow.python.layers.core.Dropout object at 0x10e144f60>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dropout.call of <tensorflow.python.layers.core.Dropout object at 0x10e144f60>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dropout.call of <tensorflow.python.layers.core.Dropout object at 0x10e144f60>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x13b71dc18>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x13b71dc18>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x13b71dc18>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x13b71dc18>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "Epoch:  0: Validation loss: 0.25939, Validation accuracy: 0.92270\n",
      "Epoch:  1: Validation loss: 0.22893, Validation accuracy: 0.93130\n",
      "Epoch:  2: Validation loss: 0.20958, Validation accuracy: 0.93720\n",
      "Epoch:  3: Validation loss: 0.19105, Validation accuracy: 0.94220\n",
      "Epoch:  4: Validation loss: 0.17505, Validation accuracy: 0.94670\n",
      "Epoch:  5: Validation loss: 0.16339, Validation accuracy: 0.94950\n",
      "Epoch:  6: Validation loss: 0.15100, Validation accuracy: 0.95390\n",
      "Epoch:  7: Validation loss: 0.14123, Validation accuracy: 0.95720\n",
      "Epoch:  8: Validation loss: 0.13186, Validation accuracy: 0.95950\n",
      "Epoch:  9: Validation loss: 0.12968, Validation accuracy: 0.96120\n",
      "Epoch: 10: Validation loss: 0.12321, Validation accuracy: 0.96280\n",
      "Epoch: 11: Validation loss: 0.11626, Validation accuracy: 0.96490\n",
      "Epoch: 12: Validation loss: 0.11864, Validation accuracy: 0.96420\n",
      "Epoch: 13: Validation loss: 0.10674, Validation accuracy: 0.96760\n",
      "Epoch: 14: Validation loss: 0.10147, Validation accuracy: 0.96880\n",
      "Epoch: 15: Validation loss: 0.10229, Validation accuracy: 0.96930\n",
      "Epoch: 16: Validation loss: 0.09544, Validation accuracy: 0.97060\n",
      "Epoch: 17: Validation loss: 0.09413, Validation accuracy: 0.97060\n",
      "Epoch: 18: Validation loss: 0.09621, Validation accuracy: 0.97030\n",
      "Epoch: 19: Validation loss: 0.09199, Validation accuracy: 0.97200\n",
      "Epoch: 20: Validation loss: 0.09008, Validation accuracy: 0.97320\n",
      "Epoch: 21: Validation loss: 0.08669, Validation accuracy: 0.97380\n",
      "Epoch: 22: Validation loss: 0.08712, Validation accuracy: 0.97360\n",
      "Epoch: 23: Validation loss: 0.08507, Validation accuracy: 0.97450\n",
      "Epoch: 24: Validation loss: 0.08310, Validation accuracy: 0.97510\n",
      "Epoch: 25: Validation loss: 0.08339, Validation accuracy: 0.97570\n",
      "Epoch: 26: Validation loss: 0.08655, Validation accuracy: 0.97470\n",
      "Epoch: 27: Validation loss: 0.08170, Validation accuracy: 0.97580\n",
      "Epoch: 28: Validation loss: 0.07928, Validation accuracy: 0.97680\n",
      "Epoch: 29: Validation loss: 0.07971, Validation accuracy: 0.97660\n",
      "Epoch: 30: Validation loss: 0.08048, Validation accuracy: 0.97690\n",
      "Epoch: 31: Validation loss: 0.08452, Validation accuracy: 0.97580\n",
      "Epoch: 32: Validation loss: 0.08100, Validation accuracy: 0.97690\n",
      "Epoch: 33: Validation loss: 0.08017, Validation accuracy: 0.97730\n",
      "Epoch: 34: Validation loss: 0.08261, Validation accuracy: 0.97760\n",
      "Epoch: 35: Validation loss: 0.07833, Validation accuracy: 0.97790\n",
      "Epoch: 36: Validation loss: 0.08339, Validation accuracy: 0.97670\n",
      "Epoch: 37: Validation loss: 0.08679, Validation accuracy: 0.97670\n",
      "Epoch: 38: Validation loss: 0.07940, Validation accuracy: 0.97780\n",
      "Epoch: 39: Validation loss: 0.08043, Validation accuracy: 0.97810\n",
      "Epoch: 40: Validation loss: 0.07796, Validation accuracy: 0.97980\n",
      "Epoch: 41: Validation loss: 0.08210, Validation accuracy: 0.97810\n",
      "Epoch: 42: Validation loss: 0.08389, Validation accuracy: 0.97750\n",
      "Epoch: 43: Validation loss: 0.08120, Validation accuracy: 0.97850\n",
      "Epoch: 44: Validation loss: 0.08152, Validation accuracy: 0.97830\n",
      "Epoch: 45: Validation loss: 0.08377, Validation accuracy: 0.97770\n",
      "Epoch: 46: Validation loss: 0.09062, Validation accuracy: 0.97680\n",
      "Epoch: 47: Validation loss: 0.08330, Validation accuracy: 0.97810\n",
      "Epoch: 48: Validation loss: 0.08681, Validation accuracy: 0.97810\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 49: Validation loss: 0.08846, Validation accuracy: 0.97670\n",
      "Epoch: 50: Validation loss: 0.08730, Validation accuracy: 0.97900\n",
      "Epoch: 51: Validation loss: 0.07956, Validation accuracy: 0.97960\n",
      "Epoch: 52: Validation loss: 0.08006, Validation accuracy: 0.97900\n",
      "Epoch: 53: Validation loss: 0.08619, Validation accuracy: 0.97900\n",
      "Epoch: 54: Validation loss: 0.08867, Validation accuracy: 0.97860\n",
      "Epoch: 55: Validation loss: 0.08767, Validation accuracy: 0.97900\n",
      "Epoch: 56: Validation loss: 0.09225, Validation accuracy: 0.97790\n",
      "Epoch: 57: Validation loss: 0.08343, Validation accuracy: 0.97890\n",
      "Epoch: 58: Validation loss: 0.08685, Validation accuracy: 0.97960\n",
      "Epoch: 59: Validation loss: 0.10281, Validation accuracy: 0.97590\n",
      "Epoch: 60: Validation loss: 0.09093, Validation accuracy: 0.97790\n",
      "Epoch: 61: Validation loss: 0.09148, Validation accuracy: 0.97830\n",
      "Epoch: 62: Validation loss: 0.08822, Validation accuracy: 0.97850\n",
      "Epoch: 63: Validation loss: 0.08832, Validation accuracy: 0.97910\n",
      "Epoch: 64: Validation loss: 0.09472, Validation accuracy: 0.97780\n",
      "Epoch: 65: Validation loss: 0.08969, Validation accuracy: 0.97870\n",
      "Epoch: 66: Validation loss: 0.09385, Validation accuracy: 0.97930\n",
      "Epoch: 67: Validation loss: 0.09270, Validation accuracy: 0.97970\n",
      "Epoch: 68: Validation loss: 0.09031, Validation accuracy: 0.97920\n",
      "Epoch: 69: Validation loss: 0.08907, Validation accuracy: 0.97890\n",
      "Epoch: 70: Validation loss: 0.08659, Validation accuracy: 0.97950\n",
      "Epoch: 71: Validation loss: 0.09671, Validation accuracy: 0.97830\n",
      "Epoch: 72: Validation loss: 0.09241, Validation accuracy: 0.97930\n",
      "Epoch: 73: Validation loss: 0.10137, Validation accuracy: 0.97790\n",
      "Epoch: 74: Validation loss: 0.08779, Validation accuracy: 0.98000\n",
      "Epoch: 75: Validation loss: 0.10460, Validation accuracy: 0.97780\n",
      "Epoch: 76: Validation loss: 0.09751, Validation accuracy: 0.97850\n",
      "Epoch: 77: Validation loss: 0.10290, Validation accuracy: 0.97830\n",
      "Epoch: 78: Validation loss: 0.09957, Validation accuracy: 0.97850\n",
      "Epoch: 79: Validation loss: 0.10337, Validation accuracy: 0.97880\n",
      "Epoch: 80: Validation loss: 0.10580, Validation accuracy: 0.97770\n",
      "Epoch: 81: Validation loss: 0.10116, Validation accuracy: 0.97910\n",
      "Epoch: 82: Validation loss: 0.09686, Validation accuracy: 0.97900\n",
      "Epoch: 83: Validation loss: 0.09842, Validation accuracy: 0.97930\n",
      "Epoch: 84: Validation loss: 0.10796, Validation accuracy: 0.97770\n",
      "Epoch: 85: Validation loss: 0.10812, Validation accuracy: 0.97710\n",
      "Epoch: 86: Validation loss: 0.09723, Validation accuracy: 0.98050\n",
      "Epoch: 87: Validation loss: 0.09719, Validation accuracy: 0.98010\n",
      "Epoch: 88: Validation loss: 0.09811, Validation accuracy: 0.98000\n",
      "Epoch: 89: Validation loss: 0.09846, Validation accuracy: 0.97950\n",
      "Epoch: 90: Validation loss: 0.10602, Validation accuracy: 0.97880\n",
      "Epoch: 91: Validation loss: 0.09493, Validation accuracy: 0.97990\n",
      "Epoch: 92: Validation loss: 0.10540, Validation accuracy: 0.97940\n",
      "Epoch: 93: Validation loss: 0.10060, Validation accuracy: 0.97940\n",
      "Epoch: 94: Validation loss: 0.10142, Validation accuracy: 0.97960\n",
      "Epoch: 95: Validation loss: 0.10550, Validation accuracy: 0.97840\n",
      "Epoch: 96: Validation loss: 0.10160, Validation accuracy: 0.97870\n",
      "Epoch: 97: Validation loss: 0.09593, Validation accuracy: 0.98010\n",
      "Epoch: 98: Validation loss: 0.09431, Validation accuracy: 0.97990\n",
      "Epoch: 99: Validation loss: 0.10533, Validation accuracy: 0.97930\n",
      "Epoch: 100: Validation loss: 0.11183, Validation accuracy: 0.97850\n",
      "Epoch: 101: Validation loss: 0.10357, Validation accuracy: 0.97920\n",
      "Epoch: 102: Validation loss: 0.10045, Validation accuracy: 0.98090\n",
      "Epoch: 103: Validation loss: 0.10677, Validation accuracy: 0.97930\n",
      "Epoch: 104: Validation loss: 0.10491, Validation accuracy: 0.97910\n",
      "Epoch: 105: Validation loss: 0.10201, Validation accuracy: 0.97980\n",
      "Epoch: 106: Validation loss: 0.10710, Validation accuracy: 0.97970\n",
      "Epoch: 107: Validation loss: 0.10988, Validation accuracy: 0.97880\n",
      "Epoch: 108: Validation loss: 0.10889, Validation accuracy: 0.97880\n",
      "Epoch: 109: Validation loss: 0.10329, Validation accuracy: 0.98030\n",
      "Epoch: 110: Validation loss: 0.11403, Validation accuracy: 0.97850\n",
      "Epoch: 111: Validation loss: 0.10568, Validation accuracy: 0.98000\n",
      "Epoch: 112: Validation loss: 0.10612, Validation accuracy: 0.97970\n",
      "Epoch: 113: Validation loss: 0.10701, Validation accuracy: 0.98000\n",
      "Epoch: 114: Validation loss: 0.10967, Validation accuracy: 0.97980\n",
      "Epoch: 115: Validation loss: 0.11624, Validation accuracy: 0.97930\n",
      "Epoch: 116: Validation loss: 0.11492, Validation accuracy: 0.97830\n",
      "Epoch: 117: Validation loss: 0.10827, Validation accuracy: 0.98020\n",
      "Epoch: 118: Validation loss: 0.11051, Validation accuracy: 0.97960\n",
      "Epoch: 119: Validation loss: 0.10243, Validation accuracy: 0.98070\n",
      "Epoch: 120: Validation loss: 0.11751, Validation accuracy: 0.97870\n",
      "Epoch: 121: Validation loss: 0.12202, Validation accuracy: 0.97690\n",
      "Epoch: 122: Validation loss: 0.10928, Validation accuracy: 0.98080\n",
      "Epoch: 123: Validation loss: 0.11003, Validation accuracy: 0.98020\n",
      "Epoch: 124: Validation loss: 0.11140, Validation accuracy: 0.98010\n",
      "Epoch: 125: Validation loss: 0.11182, Validation accuracy: 0.97970\n",
      "Epoch: 126: Validation loss: 0.12741, Validation accuracy: 0.97790\n",
      "Epoch: 127: Validation loss: 0.11361, Validation accuracy: 0.98020\n",
      "Epoch: 128: Validation loss: 0.10865, Validation accuracy: 0.98060\n",
      "Epoch: 129: Validation loss: 0.10766, Validation accuracy: 0.98020\n",
      "Epoch: 130: Validation loss: 0.10589, Validation accuracy: 0.98150\n",
      "Epoch: 131: Validation loss: 0.10902, Validation accuracy: 0.98160\n",
      "Epoch: 132: Validation loss: 0.11174, Validation accuracy: 0.98020\n",
      "Epoch: 133: Validation loss: 0.11015, Validation accuracy: 0.98140\n",
      "Epoch: 134: Validation loss: 0.11706, Validation accuracy: 0.97980\n",
      "Epoch: 135: Validation loss: 0.11935, Validation accuracy: 0.97950\n",
      "Epoch: 136: Validation loss: 0.12525, Validation accuracy: 0.97940\n",
      "Epoch: 137: Validation loss: 0.12200, Validation accuracy: 0.97950\n",
      "Epoch: 138: Validation loss: 0.12016, Validation accuracy: 0.97870\n",
      "Epoch: 139: Validation loss: 0.11826, Validation accuracy: 0.97970\n",
      "Epoch: 140: Validation loss: 0.11531, Validation accuracy: 0.97970\n",
      "Epoch: 141: Validation loss: 0.11061, Validation accuracy: 0.98120\n",
      "Epoch: 142: Validation loss: 0.12233, Validation accuracy: 0.97900\n",
      "Epoch: 143: Validation loss: 0.11452, Validation accuracy: 0.98060\n",
      "Epoch: 144: Validation loss: 0.12256, Validation accuracy: 0.97960\n",
      "Epoch: 145: Validation loss: 0.11519, Validation accuracy: 0.98050\n",
      "Epoch: 146: Validation loss: 0.12227, Validation accuracy: 0.97900\n",
      "Epoch: 147: Validation loss: 0.11433, Validation accuracy: 0.97990\n",
      "Epoch: 148: Validation loss: 0.11786, Validation accuracy: 0.98070\n",
      "Epoch: 149: Validation loss: 0.12990, Validation accuracy: 0.97920\n",
      "Epoch: 150: Validation loss: 0.11166, Validation accuracy: 0.98050\n",
      "Epoch: 151: Validation loss: 0.12575, Validation accuracy: 0.97880\n",
      "Epoch: 152: Validation loss: 0.12432, Validation accuracy: 0.98000\n",
      "Epoch: 153: Validation loss: 0.13570, Validation accuracy: 0.97810\n",
      "Epoch: 154: Validation loss: 0.11073, Validation accuracy: 0.98080\n",
      "Epoch: 155: Validation loss: 0.11693, Validation accuracy: 0.98070\n",
      "Epoch: 156: Validation loss: 0.11548, Validation accuracy: 0.98130\n",
      "Epoch: 157: Validation loss: 0.12476, Validation accuracy: 0.97920\n",
      "Epoch: 158: Validation loss: 0.12082, Validation accuracy: 0.97950\n",
      "Epoch: 159: Validation loss: 0.11222, Validation accuracy: 0.98070\n",
      "Epoch: 160: Validation loss: 0.12270, Validation accuracy: 0.97980\n",
      "Epoch: 161: Validation loss: 0.11590, Validation accuracy: 0.98110\n",
      "Epoch: 162: Validation loss: 0.12300, Validation accuracy: 0.98050\n",
      "Epoch: 163: Validation loss: 0.12461, Validation accuracy: 0.98000\n",
      "Epoch: 164: Validation loss: 0.11425, Validation accuracy: 0.98070\n",
      "Epoch: 165: Validation loss: 0.12184, Validation accuracy: 0.97970\n",
      "Epoch: 166: Validation loss: 0.11741, Validation accuracy: 0.98090\n",
      "Epoch: 167: Validation loss: 0.11078, Validation accuracy: 0.98160\n",
      "Epoch: 168: Validation loss: 0.12008, Validation accuracy: 0.98070\n",
      "Epoch: 169: Validation loss: 0.11671, Validation accuracy: 0.98130\n",
      "Epoch: 170: Validation loss: 0.12718, Validation accuracy: 0.97970\n",
      "Epoch: 171: Validation loss: 0.12851, Validation accuracy: 0.97990\n",
      "Epoch: 172: Validation loss: 0.11319, Validation accuracy: 0.98190\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 173: Validation loss: 0.11461, Validation accuracy: 0.98100\n",
      "Epoch: 174: Validation loss: 0.12407, Validation accuracy: 0.97980\n",
      "Epoch: 175: Validation loss: 0.12596, Validation accuracy: 0.98020\n",
      "Epoch: 176: Validation loss: 0.11093, Validation accuracy: 0.98130\n",
      "Epoch: 177: Validation loss: 0.11323, Validation accuracy: 0.98150\n",
      "Epoch: 178: Validation loss: 0.12516, Validation accuracy: 0.97900\n",
      "Epoch: 179: Validation loss: 0.11838, Validation accuracy: 0.98140\n",
      "Epoch: 180: Validation loss: 0.12092, Validation accuracy: 0.98150\n",
      "Epoch: 181: Validation loss: 0.11630, Validation accuracy: 0.98200\n",
      "Epoch: 182: Validation loss: 0.12878, Validation accuracy: 0.98080\n",
      "Epoch: 183: Validation loss: 0.12916, Validation accuracy: 0.97960\n",
      "Epoch: 184: Validation loss: 0.12557, Validation accuracy: 0.97990\n",
      "Epoch: 185: Validation loss: 0.11492, Validation accuracy: 0.98080\n",
      "Epoch: 186: Validation loss: 0.11970, Validation accuracy: 0.98150\n",
      "Epoch: 187: Validation loss: 0.11987, Validation accuracy: 0.98120\n",
      "Epoch: 188: Validation loss: 0.12580, Validation accuracy: 0.98040\n",
      "Epoch: 189: Validation loss: 0.13940, Validation accuracy: 0.97820\n",
      "Epoch: 190: Validation loss: 0.12253, Validation accuracy: 0.98100\n",
      "Epoch: 191: Validation loss: 0.13424, Validation accuracy: 0.97960\n",
      "Epoch: 192: Validation loss: 0.11628, Validation accuracy: 0.98130\n",
      "Epoch: 193: Validation loss: 0.12335, Validation accuracy: 0.98110\n",
      "Epoch: 194: Validation loss: 0.12422, Validation accuracy: 0.98080\n",
      "Epoch: 195: Validation loss: 0.12638, Validation accuracy: 0.98040\n",
      "Epoch: 196: Validation loss: 0.13038, Validation accuracy: 0.97960\n",
      "Epoch: 197: Validation loss: 0.13296, Validation accuracy: 0.97960\n",
      "Epoch: 198: Validation loss: 0.12603, Validation accuracy: 0.98120\n",
      "Epoch: 199: Validation loss: 0.11239, Validation accuracy: 0.98180\n",
      "Pruning smallest elements (baseline):\n",
      "0.00, 0.9818\n",
      "0.10, 0.982\n",
      "0.20, 0.9817\n",
      "0.30, 0.9812\n",
      "0.40, 0.9811\n",
      "0.50, 0.9807\n",
      "0.60, 0.9792\n",
      "0.70, 0.9749\n",
      "0.80, 0.9446\n",
      "0.90, 0.5514\n",
      "0.95, 0.2315\n",
      "0.96, 0.2002\n",
      "0.97, 0.1578\n",
      "0.98, 0.1604\n",
      "0.99, 0.1009\n",
      "0.99, 0.101\n",
      "0.99, 0.0982\n",
      "1.00, 0.0982\n",
      "1.00, 0.098\n",
      "Pruning using signal-to-noise from Bayes:\n",
      "0.00, 0.9818\n",
      "0.10, 0.9818\n",
      "0.20, 0.9818\n",
      "0.30, 0.9818\n",
      "0.40, 0.9822\n",
      "0.50, 0.9822\n",
      "0.60, 0.9816\n",
      "0.70, 0.9804\n",
      "0.80, 0.9751\n",
      "0.90, 0.9398\n",
      "0.95, 0.7219\n",
      "0.96, 0.5809\n",
      "0.97, 0.3905\n",
      "0.98, 0.1653\n",
      "0.99, 0.1135\n",
      "0.99, 0.1135\n",
      "0.99, 0.1032\n",
      "1.00, 0.1032\n",
      "1.00, 0.098\n"
     ]
    }
   ],
   "source": [
    "runs=1\n",
    "xx = []\n",
    "yy_aa = []\n",
    "yy_bb = []\n",
    "accs = []\n",
    "ww = []\n",
    "ss = []\n",
    "for _ in range(runs):\n",
    "    x, ya, yb, acc, weightvalues, sqrtsigmavalues, _ = train(X_train, Y_train,\n",
    "                                                             X_test, Y_test,\n",
    "                                                             num_epochs=200, \n",
    "                                                             learning_rate = 0.001, l2 = 0.0,\n",
    "                                                             dropout= 0.2, beta2=0.999, laplace_mle=False, \n",
    "                                                             no_bias_init=False,\n",
    "                                                             prec=1/(0.5**2), size=60000, \n",
    "                                                             act='sigmoid')\n",
    "    xx.append(x)\n",
    "    yy_aa.append(ya)\n",
    "    yy_bb.append(yb)\n",
    "    accs.append(acc)\n",
    "    ww.append(weightvalues)\n",
    "    ss.append(sqrtsigmavalues)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x13b70fac8>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAD4CAYAAAAQP7oXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3hc9ZX4//eZGWlGddSb1d27jW3caAECBgKGBAiEDSRkYUnbZDfZhGTzTbK7WXbJ7mYT8ssCISGkUAIJPZQkgG2KMTbuvciWrN67RtJoPr8/pnjUR7KlUTmv59Fj6c6dO0dX43vmfNoVYwxKKaVUKCzhDkAppdTkoUlDKaVUyDRpKKWUCpkmDaWUUiHTpKGUUipktnAHMNZSUlJMfn5+uMNQSqlJ48MPP6w1xqQO9NiUTxr5+fns2LEj3GEopdSkISLFgz2mzVNKKaVCpklDKaVUyDRpKKWUCpkmDaWUUiGbsklDRK4VkZ83NTWFOxSllJoypmzSMMa8ZIy52+l0hjsUpZSaMqZs0lBKKXXuadIYxEObT/DXg1XhDkMppSYUTRqD+NW7J/nzwcpwh6GUUhOKJo1BpMbZqW7pDHcYSik1oWjSGERanIMaTRpKKdWLJo1BpMZqpaGUUn1p0hhEWrydutZOejx6D3WllPLTpDGI1Dg7HgN1bVptKKWUnyaNQaTF2QG0X0MppYJo0hhEqi9paL+GUkqdoUljEGlxDkArDaWUCqZJYxCp2jyllFL9aNIYhCPCSpzDpklDKaWCaNIYgndWuCvcYSil1IShSWMIaXF2rTSUUiqIJo0hpMY5dPSUUkoFmbJJ41zcuU8rDaWU6m3KJo1zcee+1Dg77V09tHa6z2FkSik1eU3ZpHEupMd7h91WNmlnuFJKgSaNIWUnRgNQ2tAe5kiUUmpi0KQxhOzEKABKGzrCHIlSSk0MmjSGkB7nIMIqmjSUUspHk8YQLBZhRkKUNk8ppZSPJo1hZCdGa6WhlFI+mjSGkZ0YpUlDKaV8NGkMIzsxitrWTjq6esIdilJKhZ0mjWH4h92WNWq/hlJKadIYhn/Y7WltolJKKU0aw8lJ8k/w06ShlFKaNIaRGmsn0mrRYbdKKYUmjWFZLMKMxChK67XSUEopTRohyE2Kpri+LdxhKKVU2GnSCEFecjTFde0YY8IdilJKhdWkTBoicr2IPCIiL4jIFWP9erlJ0bS43DS0d4/1Syml1IQWUtIQkQQR+YOIHBaRQyKydjQvJiKPiki1iOwf4LENInJERI6LyL1DHccY87wx5i7gM8AnRxPLSOQlxwBQXKdNVEqp6S3USuMnwGvGmHnAUuBQ8IMikiYicX22zRrgOI8BG/puFBEr8DPgKmABcKuILBCRxSLycp+vtKCnfsf3vDGVl+wddltSryOolFLTm224HUQkHrgI76d6jDFdQFef3S4GPi8iVxtjXCJyF3ADcHXwTsaYLSKSP8DLnA8cN8YU+V7zKWCjMeY/gI8NEJMA/wm8aozZOdzvcLZyfXM1ius0aSilprdQKo1CoAb4lYjsEpFfiEhM8A7GmGeA14CnROQ24E7g5hHEMQM4HfRzqW/bYL4MXA7cKCL3DLSDiFwrIj9vamoaQRgDc0RYyYh3aNJQSk17oSQNG3Ae8KAxZjnQBvTrczDG/BBwAQ8C1xljWkcQhwywbdChSsaYB4wxK4wx9xhjHhpkn5eMMXc7nc4RhDG43ORo7dNQSk17oSSNUqDUGLPN9/Mf8CaRXkTkQmAR8BzwvRHGUQrkBP2cDZSP8BhjKi8pmmLt01BKTXPDJg1jTCVwWkTm+jZdBhwM3kdElgOPABuBzwJJIvKDEcSxHZgtIgUiEgncArw4guePubzkaGpaOmnvcoc7FKWUCptQR099GXhcRPYCy4D7+jweDdxkjDlhjPEAdwDFfQ8iIk8CW4G5IlIqIp8DMMa4gS8Br+MdmfW0MebAaH6hseIfdqsjqJRS09mwo6cAjDG7gZVDPP5un5+78VYeffe7dYhjvAK8Eko84eAfdltc1868jPgwR6OUUuExKWeEh0Nekk7wU0opTRohckZH4IyK0GG3SqlpTZPGCOQlR2ufhlJqWtOkMQK5SdFaaSilpjVNGiOQnxxDWWMH3T2ecIeilFJhoUljBHKTo+nxGMr0fuFKqWlKk8YI5PkXLtR+DaXUNKVJYwQCE/x02K1SaprSpDECaXF27DaLdoYrpaYtTRojYLEIecnRnKzVSkMpNT1p0hihBZnx7C8/+3t0KKXUZKRJY4SW5iRQ1dxJZZMr3KEopdS406QxQkuyEwDYU9oY5kiUUmr8adIYoYVZ8dgswp7TmjSUUtOPJo0RckRYmZsRx95S7ddQSk0/mjRGYWlOAntKG/F4Br2NuVJKTUmaNEZhabaTFpebUzrJTyk1zWjSGAX/nftO1GjSUEpNL1M2aYjItSLy86amc9/3MCMxCoCyBp0ZrpSaXqZs0jDGvGSMudvpdJ7zYyfHROKIsFCqq90qpaaZKZs0xpKIkJ0YrUlDKTXtaNIYpRkJUZQ1atJQSk0vmjRGKTsxilLt01BKTTOaNEYpOzGahvZu2jrd4Q5FKaXGjSaNUQqMoNImKqXUNKJJY5SyfUlDm6iUUtOJJo1ROpM0tNJQSk0fmjRGKSXGTqTNQpkmDaXUNKJJY5QsFiE7IUorDaXUtKJJ4yxkJ0VTUq99Gkqp6UOTxlkoTInhZG0bxugS6Uqp6UGTxlkoSImhtdNNTUtnuENRSqlxoUnjLBSmxgC6RLpSavrQpHEWClNjASiqbQ1zJEopNT40aZyFzHgHjggLRVppKKWmCU0aZ8FiEQpSYimq0UpDKTU9aNI4S/4RVEopNR1o0jhLhakxnG7ooMvtCXcoSik15jRpnKXC1Bh6PIaSeq02lFJTnyaNszTTN4LqWJX2ayilpj5NGmdpbkYcEVZhb1lTuENRSqkxp0njLNltVuZmxLGvVJOGUmrq06RxDiyekcDe0kZdg0opNeVp0jgHlmQ7aXa5Ka7TFW+VUlObJo1zYEm2E0D7NZRSU96kTBoicr2IPCIiL4jIFeGOZ056HJE2C/tKG8MdilJKjamQk4aIWEVkl4i8PNoXE5FHRaRaRPYP8NgGETkiIsdF5N6hjmOMed4YcxfwGeCTo43nXImwWliQGc8e7QxXSk1xI6k0vgIcGugBEUkTkbg+22YNsOtjwIYBnm8FfgZcBSwAbhWRBSKyWERe7vOVFvTU7/ieF3Yr8xLZfboRV3dPuENRSqkxE1LSEJFs4BrgF4PscjHwgog4fPvfBTzQdydjzBagfoDnnw8cN8YUGWO6gKeAjcaYfcaYj/X5qhav+4FXjTE7Q/kdxtr62Sl0uT3sONUQ7lCUUmrMhFpp/Bj4BjDgAkvGmGeA14CnROQ24E7g5hHEMQM4HfRzqW/bYL4MXA7cKCL3DLSDiFwrIj9vahqfJqPz85OIsArvHK8dl9dTSqlwGDZpiMjHgGpjzIdD7WeM+SHgAh4ErjPGjGRdDRnokEO81gPGmBXGmHuMMQ8Nss9Lxpi7nU7nCMIYvRi7jeW5ibyrSUMpNYWFUmmsB64TkVN4m40uFZHf9d1JRC4EFgHPAd8bYRylQE7Qz9lA+QiPEXYXzEphf3kTDW1d4Q5FKaXGxLBJwxjzLWNMtjEmH7gFeNMY8zfB+4jIcuARYCPwWSBJRH4wgji2A7NFpEBEIn2v8+IInj8hrJ+VgjGwtagu3KEopdSYOFfzNKKBm4wxJ4wxHuAOoLjvTiLyJLAVmCsipSLyOQBjjBv4EvA63hFaTxtjDpyj2MbN0mwnsXab9msopaYs20h2NsZsAjYNsP3dPj934608+u536xDHfgV4ZSTxTDQ2q4U1hcnar6GUmrIm5YzwieyCWckU17Vzul7XoVJKTT2aNM6xC2anAGi1oZSakjRpnGMzU2NJj7drv4ZSakrSpHGOiQjrZqaw7eRAE9+VUmpy06QxBpZkO6lp6aS62RXuUJRS6pzSpDEGFs3wzkLfX66r3iqlphZNGmNgfmY8InCgrDncoSil1DmlSWMMxNptFCTHsL+8CWMM3T0DrvOolFKTjiaNMbJwhpP9Zc3828uHuOJ/t4Q7HKWUOic0aYyRhVnxlDV28Nh7JzlZ20Z7lzvcISml1FnTpDFGFmV5O8M9vgXeyxt1JJVSavLTpDFGFmc7iXPYuH5ZFgAVTR1hjkgppc7eiBYsVKFzRkWw6/99lPJGF8/vLqdCKw2l1BSglcYYslktpDvtAJRrpaGUmgI0aYwxu81KSqxdKw2l1JSgSWMcZCU4tNJQSk0JmjTGQabTQUWTVhpKqclPk8Y4yHRGUdHYgTEm3KEopdRZ0aQxDrISHLR19dDs0gl+SqnJTZPGOMh0RgFQqU1USqlJTpPGOMhKcAA67FYpNflp0hgH/kpDh90qpSY7TRrjIC3OTqzdxr6yxnCHopRSZ0WTxjiwWS1cODuFNw9X6wgqpdSkpkljnFw6L42q5k4OlOvd/JRSk5cmjXHykXlpiMAbh6rDHYpSSo2aJo1xkhJrZ1lOAm8crgp3KEopNWqaNMbR1Ysy2VvaxF8PauJQSk1OmjTG0e3r8pifGc+9z+6ltrUz3OEopdSITcqkISLXi8gjIvKCiFwR7nhCZbdZ+ckty2jq6ObhzSfCHY5SSo3YsElDRBwi8oGI7BGRAyLyL6N9MRF5VESqRWT/AI9tEJEjInJcRO4d6jjGmOeNMXcBnwE+Odp4wmFOehznFyTx9rHacIeilFIjFkql0QlcaoxZCiwDNojImuAdRCRNROL6bJs1wLEeAzb03SgiVuBnwFXAAuBWEVkgIotF5OU+X2lBT/2O73mTyrqZKRyubNEmKqXUpDNs0jBerb4fI3xffWeoXQy8ICIOABG5C3hggGNtAeoHeJnzgePGmCJjTBfwFLDRGLPPGPOxPl/V4nU/8KoxZmeov+xEsW5mMgBbT9SFORKllBqZkPo0RMQqIruBauAvxphtwY8bY54BXgOeEpHbgDuBm0cQxwzgdNDPpb5tg/kycDlwo4jcM0jM14rIz5uamkYQxvhYPMNJnN3Ge76k8d7xWr793D463T1hjkwppYYWUtIwxvQYY5YB2cD5IrJogH1+CLiAB4HrgqqTUMhALztEPA8YY1YYY+4xxjw0yD4vGWPudjqdIwhjfNisFlYXJrH5SDU/e+s4d/zqA57YVsK+0omX4JRSKtiIRk8ZYxqBTQzcL3EhsAh4DvjeCOMoBXKCfs4Gykd4jEnlsvnplDe5+K/XjzA/Mx5AlxhRSk14tuF2EJFUoNsY0ygiUXibhe7vs89y4BHgGuAk8DsR+YEx5jshxrEdmC0iBUAZcAvwqdB/jcnnllU5XDovjR6PISPewcp//ysHyrXSUEpNbKFUGpnAWyKyF+/F/S/GmJf77BMN3GSMOWGM8QB3AMV9DyQiTwJbgbkiUioinwMwxriBLwGvA4eAp40xB0b7S00GIkJ6vIOshCgsFmFhVrxWGkqpCW/YSsMYsxdYPsw+7/b5uRtv5dF3v1uHOMYrwCvDxTNVLciK51fvnKLL7SHSNinnXCqlpgG9Ok0QC7OcdPV4OF49kvEDSik1vjRpTBALAp3h2q+hlJq4NGlMEAUpMURFWLVfQyk1oWnSmCCsFmFpjpMdxQNNmFdKqYlBk8YEsm5mCgfKm2ls7wp3KEopNSBNGhPIupnJGAPvF2m1oZSamDRpTCBLshOIjrSy9YQum66Umpg0aUwgkTYLq/KTAgsZKqXURKNJY4JZNzOZY9WtVDR1hDsUpZTqR5PGBLNhUQZWi/DoOyfDHYpSSvWjSWOCyUuOYeOyLH77fjHP7yrj87/7UO/wp5SaMDRpTEBfvnQ2XW4PX/39bl7dX8lzO8vCHZJSSgGaNCakgpQYvnXVfP7+stnMy4jj1f0V4Q5JKaWAEFa5VeFx10WFAERahf/+81Eqm1xkOB1hjkopNd1ppTHBbViUCcDrByrDHIlSSmnSmPBmpcUyOy2W53drv4ZSKvw0aUwCt63OZVdJI9tPeZcXqW5xsea+N9hWpJMAlVLjS5PGJHDzqhwSoyN4aNMJAN48VE1ls4s/7dMOcqXU+NKkMQlER9r4zLoC3jhczeHKZrYcqwHg3eO6RpVSanxp0pgkbl+bR3Sklf976wRvH6vFEWHhRE0bVc2ucIemlJpGNGlMEokxkdx6fi4v7imnxeXmzvUFALynK+IqpcaRJo1J5HMXFGCzCBaBuy4sxBkVwXvHtTNcKTV+NGlMIlkJUXzuggKuW5pFYkwkawuTee9EHcaYXvu9sLuMa3/6Du4eT5giDV1JXTuHKvS+6EpNFpo0JplvXT2fH9+yHID1s5Ipa+ygpL691z4v7i5nX1kThypawhHiiPzna4f40hM7wx2GUipEmjQmsbUzUwB4N6iJqsdj+MA3n2PbyYnfdFXb0kVJfTs9HjP8zkqpsNOkMYnNTI0hPd7Ou0Gd4UcqW2hxuQH44OTEv9d4U0c33T1GR4EpNUnogoWTmIiwfmYKm4/WUFLXTmlDO0ervE1S62cls/1UPR6PwWKRMEc6uMaOLgBK6tvJSogKczRKqeFopTHJrZ2ZTF1bFx/938186hfb+MU7J8lOjOL6ZTNoaO/meE1rv+d8/8UDXPY/m7jh/96lsb1rRK/X0dXT71a0X3t6D28drh5V/E0d3QCc7tMvo5SamDRpTHIXzE4h0mphQVY8K/MSKW3o4PyCJFYXJAOw9UTvfo1Odw9PbCuhq8fDrpJG3hnhrPL7XzvMNQ+8E+iDqGnp5I87S/nzwaoRx+7q7sHV7R3hdbpB74k+3p7ecVpXFVAjpkljkst0RrH5G5fw9N+t5Rd3rOTy+WnceF42OUlRLMiM58FNJ2h2dQf2P1TRQlePh3+6ch5REVZ2nGoI+bWMMfzlYBX1bV2cqmvzHc87XLayaeQX/eaOM3FN1Epj89EaTtW2hTuMMXH/q4f59Xunwh2GmmQ0aUwBmc4oIqwWEqIj+cUdq1g3KwUR4b6PL6a6xcX9rx4O7LvndCMAK/MSWZaTwI7i0DvLj1e3UtboTQ77y5qAM0mjomnkHdlNEzxpVDW7+Nxj23l4y4lwh3LONbu6qWvrCgyaUCpUmjSmsGU5Cdy5voDHt5Xw0p5yAHafbiQtzk6m08Gq/EQOljfT2tn/wtHjMf0mB2464l0o0WoRDpR7k8XhSm/He+UoRj/5k0Z6vJ3TDRMvaTy+rQS3x1DfNrJ+n8mgpM57vls6u4fZU6neNGlMcf+0YS7n5yfx9Wf2sKukgT2nG1mak4CIsDI/CY+BXSX9m6i+/swePvWLbb22bTpazey0WBZlxferNBrbu3F194wotsZ27wVr8QwnVc2dI37+WOpye3hiWwlwJs6ppNiXNJo7tNJQI6NJY4qz26w89OkVpMXbues3OyiqbWNZTgIAy3MTsAj9+jUqmjp4YXcZ20/VB0ZXtbi62X6ygUvmprJwhpP9ZU10uns4Xt1KRrz33uWVI2yi8lcai2Y4ASgdoDP8ZG0bne6zSyZHq1r6LbXi9+Kecr44wIz0Px+spLa1k8ToiF7NaFOFv0+qxTX1fjc1tjRpTANJMZE8escqOn0jlZb7kkacI4JFM5z84cNSals7A/s/sa0EjwFjzkwQfH5XGV09Hq5ZksWiLCfNLjebjtTg9hgumZsKDN+v0ezq5l9fOhi4UAWSRpY3aZTU9+5wbu10s+HHW3h4c9GIft8WVzc/ePkg7V1uDlc2c8X/bhn0hlV/+LCUP+2toKlPNbHndCN2m4VL56VPyUrD3zzV7HIPmFBd3T38+58O9jsvSmnSmCZmp8fx0KdXcNGcVJbnJga2/9vGRdS2dnLPbz+ku8dDl9vDkx+c5sLZKdhtFradrMcYw2/fL2bxDCfLchJYNCMegAd9dxK8ZG4aAJXNQ4+gen1/JY++e5K/+Ibn+pPGqvwk4h02HtlystcF7HBFM51uD2+OcA7Im4er+cU7J9lWVB8Y+fTszv73WPd4TKBp7khV73W6imraKEiJITk2MjABcSrxVxo9HkN7V/9K7sPiBh55+ySbjo5u/s142VnSQHmjDtceT5o0ppH1s1L4zZ3nExVpDWxbmpPAv21cxI7iBrYV1bOntJHa1k5uW53HebmJvF9Uxwcn6zla1cqn1+QBMDcjjpTYSHafbiQ93s66Wd45IcNVGrt8I7f81UtTRzdxDhvO6Ai+edU8thbV8fzuMxf3g77+kr2ljSF94q1p8VZL/lnx5U0dgZi2HK2hvq2LqmZXIDGdqGkNjB46Utl7pd2iWm/ScEZF4Or2TKj+llC5ezw0DNKJX1zXjn+hgIFGUPkXwaxu7uz32ETyuce289M3j4c7jDHx478e5bldpeEOox9NGooL53gXPjxZ18ZJ3yfz+ZlxrC5M4mBFM996dh/OqAiuXZoFePtJ3r33UvZ89wre/salxDsiiHfYqBomaews9n6qD04azqgIAG5dlcvy3AS+89x+/uqrRA76Rmh5zPA3m9p8tIbV9/2V49UtHKn0zoKvaHQFRnW5PYa7f7OD1fe9wed/t5O2Tjc7fVWG1SK9Ko3uHg8l9e0UpsaQEB0RiHWk9pU28ezO8P2n/+U7J7nkvzfR3WcUnKu7h8pmF7PT4gB6zePx8yeNibwmWFNHNw3t3VRP4BjPxu/eL+bF3eXhDqMfTRqK9DgHdpuFkro2iuvasFqErIQo1hQmYww0tHfx8KdX9KpQ7DYrzugIIm3et1CmM2rASsMYw/HqFlo73RytaiEpJpKi2jaqW1w0dXQHLsoWi/DgbSsoTI3lrt/u4I1DVRysaGZ1QRJxdhtbjg2dNLYV1eEx8M6x2l6VRmWTi5ykKGanxbKjuIE1hUn8+WAltz7yPu+dqCMhOoLlOQkcqTyTNPyr7hamxJIQFQmMbgTVT944xj8+vYfndw3cNPapR97nDx+OLqk8tPkE33p235D7bDpSQ1NHd78BCv6E4B+A0DxAQvTvM5qh1OOlzDdwonYKDonudPdQ29o1IYd7a9JQWCxCXnI0p+raKa5rJzvRO1nw/Pwk/nXjQl780gWsKUwe8hjpTseAF5j/23SCy3+0hfteOYTHwB1r8wFvtdHY3hWoNAAynA6e/ru1ZCdG8fDmIg5XtrAk28namcm8faxm0BFQAPt8Q4A3H60JXPAqGl1UNrnIjI/iPz+xhJ/csown71rDg3+zgn1lTbywu5zlOQnMy4zjSOWZEVZFNd5qqyCo0hjpGl1wZgLkN/+4t1dSAthf3sR7J+r45TsnQzqWu8dDp7snEONfDlbx0p7yQc9Jl9vDrtPeSqrvqDT/cNvFvr6pgZqnTp/D5qknPyhh89Gasz5OX6W+uT11rRO7CW00/Oe9TpOGmqjykmMormujuK6dvOQYwJtMbl+bT05S9LDPz4x39Ks0Shva+embxwACcx7+Zk0u0ZFWPjhZ36t5yi8q0sotq3L54FQ9XW4PC7LiuWx+GqUNHewsaey1b0dXDztOeTvq/RfoTb6LU0yklYqmDiqbXWQ4HazIS2TjshmICFcuzODrV8wFYEVeInPT42h2uQNJr8i3yOPMlNhAfI0DfBo/UN7E8n/9MycGWBSypqWTymYX91w8k64eD6/u7z16641D3g7mQxXNAz4/2MHyZhZ893Xmfuc1vvLUbsB7UW/tdAf6cfraX94UWNerrE9HsT8hLPRXGkM1T7WcXaXR3uXmey8e4IE3jp3VcfyMMVzzwNv89v3iwO9V1zrxLqxny98sqJWGmrDykqIprmvnVG0b+cnDJ4m+MpwOals76XT3cKSyhU//chu3+SYHPnCr906Ds9JiSY61syo/iS1Ha3xJI7LfsW5akY3V10u7INPJNUuyiI608tQHJYF9PB7Dl5/cxY0PbeXFPeU0tHczPzMe/wfvdbNSKG/yVhoZTke/1/jCJTP5yS3L+Js1ecxJ97bt//ufDnHvH/dyuLKF5JhInNERZ/o0BmieemZHKQ3t3QMu+re/3JvELpmbSm5SNMeqeyeGNw5XUZgSgwi8vKd3QvF4TK+bUv31UBXdHg9Lsp28X1SHq7uHal+yGGgVY+h9L5XSPrPtK5o6sNss5Pn+zs19Ko2mjm4a27uJtFp6DRwYjfeO19Hl9rC3tLHfYIJnd5byZNDf1O/Nw1X8dJAk09DezYHyZjYdrg5UUB3dPbR3TaxJiq/tr+Cxd0OrIgfi/wDT3tUz4QZhaNJQAOSlxNDp9tDS6Q5UGiOxJNuJMbD5SA0PbznB9lP1pMc5+OGNS7luaRbf3DCPuy8qBGDDogxO1bVT29rVr9IASIt3cPn8NBwRFgpTY4i129i4LIuX91YE5nj836bj/PVQFRaB+145BMCd6/MBsNssrC5IosvtodPtCUw+DCYibFw2g4ToSOZlxGMReHlvBU9tP81zu8ooSPGeg4RoX59Gn2G3Ho/hFd/cj919KiCAA77KZ0FWPLPTYjkW1NFe2eRif1kzN67MZlV+Ei/vPdPZeaC8iQt/+Bb/8tKBwLZ3j9eyMCueDYsyqG7p7NXUdaJm4MUUt5+spzDFe5Ouvs1T5U0ushKiiHd4z33fPg1/JbI0x4mr23NWs8bfPOKtqLp7DHtLm3o99putxfx8S/85OE9sK+F//3q01+CDLUdraGjrClRA+8ubeiXDvtXG3tLGwECKcHjig9M88vZZJI2gqn2iNVFp0lAAvaqL0VQaF89JJS3OzqPvnuSVfRV8/Lxsnr5nLdf5Rlx9/pKZ3LwyB4ArF2YEKomBkgbAD65fzON/u4YIq/ctesuqXDq6e3h2ZxmVTS5+8sYxPrYkk5tX5lDV3InVInxsSRaJ0RHMTo8lO/HM75A5QKURzBkdwZN3reHNr13MPRfPBKAw1Zs0YiKt2CzSryN8R3ED1S2dxERa2V3qTRrVLS5K6tpxdfewr6yJgpQY4h0RzE6P42RtW2AUk3/eyWXz0rliQTrHqlupbnFxuLKZmx7aSlljB8/uLMPV3UNHVw+7ShpZNzOFwpRYgF79Ayeq+1ca7h4PO4obWJWfRHZidKDD2C7scCwAABtpSURBVK+8sYNMpwNHhJVIq6Vf85Q/aazMTwJG10TV3uWmu8fDW4erWV3gPc72U70Xx6xudnG6vr3fGmdFNW14DLxf5F3Wv6mjmzt+9QG/fOckxb75JVXNnew53USE1fs+6nth/fZz+7jl51sDkxjHW11rJzUtnaOu0oJHrdVPsOY3vXOfAiA/qLrIG0XSsFktfGJFdmDC3y2rcgbdNykmknUzk3n7WO2gSSM1zk5qnD3w85JsJyvzEvnpm8c4UtWCx8A3N8yjrq2Lp7afZnZaLFGRVr65YR6xDluvRJE+TNIAWO3r6P/6FXPwGMMVC9IBb0WSEB3Rr0/jT3vLsdss3L4unwc3nWDz0Ro+86sPMAbS4uy4PYZ1M73HnJ0WS3ePobiujVlpcTy/q4zClBjmpMdS7ltS/nR9Ox+cbKC9q4f7bljMt5/bx9vHanFEWOjq8bBuZjIzfHc29CeN9Hj7gP0hf9pXQVNHN1csTKdjd0+gQ9yvotHFBbO9w6zjo2z9OsL9n+ZX5nkngVY1uwJNeMOpbnZx/2tHeGlPOTF2Kw3t3fzD5XOob+tiR1DS8HgM1S2duD2G8kYXub73nH+4M3grrCsXZnCytg1jvP0/jogzn3Mrm10syXayt7SpX2d4eaOLZpebe373Ic9+YR2OCCvjqba1k64eb5XmjB74PT6UyqABCHVtE6ujf1JWGiJyvYg8IiIviMgV4Y5nKsh0OrBZBBF6fUofCX8lMS8jjsW+TtbBXL04Exi80uhLRPjutQuobe3iiW0lXLM4k5ykaJZmO1lbmMxl872z0m85P5ePLckiM+FMohiu0ghms1r49tXzA5+y/TEG92m0d7l5YU85l81PCySGb/5hL/GOCO7/xGIirBbq27oC58B/wT1W1UpRTSsfnKrnppU5iAi5vkEGJfXtlNS3kRwTyU0rs3FGRfDKvgreO1GHzSKsyk8iNzkaq0XYVdJApM3C6oJk76dyj6HDN6vbGMNDm4uYlRbLR+amkZ0YRUWji+2n6vni4zt9/SEusnznJN4R0a95qqS+ncToCGaleSubUNcU63J7uPu3H/Ly3nJuXJnNkuwEUuPsXDo/jZX5SewobqCoppUut4e6ti7cvn6bk3Vt/H57CT/6y1FK6ttxeww2i/COb5i1f2DC4coWiuvaiXOc+ay7NNu7JE5w85Sruydw/g9WNId0o6meoHM4EFd3T8hVg8djAvHUDDCyy+MxfOmJnYGVEQZS2dRBerz3Q1NDexcPbz7B794vHnDf1k43n3rkfa76ydt874X9IcV4NoZNGiKSIyJvicghETkgIl8Z7YuJyKMiUi0i/X4zEdkgIkdE5LiI3DvUcYwxzxtj7gI+A3xytPGoM2xWCzlJ0WQ5o0b9qawgJYZvbpjHd65ZgMjQ9yW/dmkWn1mXH7johmJJdgI3rcgGCPSPiAhP3r2Gf7pyXq99U2LsRFgFi0BqrL3fsUYiIbr3UiJPbz9NY3s3n7uggCW+i1Zls4vb1+bxyVW5PPeFdXx2fT43LJ8BwMzUWETgaFUrz3xYitUifOI872MzEqIQgZK6Dk7VtpOXHE2E1cJHF6Tzyr4KHnv3FCvyEomx27DbrOQkRuExkJ3onXtS1tjBHb/6gI/89yaqW1xsOlrDoYpm7r6oEItFmJEYhdtj+N4LB/jTvgq2HK3BYyDTV7XEOWy9OsKNMewqaSQ3OYZ0X19Q9SAjtPr6j1cPsft0Iz/+5DLuu2Exv77zfLb/8+WkxNpZXZBEi8vNpf+zmX98enev5pfiujZ+9e4pHt58gsMV3v6aqxZnUlTbRlljR2AIdFljBwcrmpmXERdItkuyvYm5NujTuH+4qv/8HyxvxuMxPLerdNDFL/+/N49z+Y82D5oYbvi/9/j4g+/1q2h2ljSwr09fTVNHdyAhDjS67XRDOy/vreCrT+0adORcZbOL+ZneIdF1rV089t4pfr/99ID7bjlaw3sn6mhq7+LJD073GkQxFkKpNNzA14wx84E1wBdFZEHwDiKSJiJxfbbNGuBYjwEb+m4UESvwM+AqYAFwq4gsEJHFIvJyn6+0oKd+x/c8dQ6sn5XM+lmhX8QH8vlLZgaaPoYSa7fx/esWkhjTf/TUUP514yL+cM/awMS0wVgsQobTQWqcHZv17ArqhKiIQJ+Gu8fDL945yYq8RFbkJeGMiqAwNYZIm4U71uUD3o787127kDTfRTcq0kpOYjTbTtbxhw9LuWROauAxR4SVjHgHJfXtFNe1BZoJP7kqh0ibhasXZ/LfNy0NxDIz1fvpPycxmpm+SuDtY7XUtHZy52Pb+fsndpGTFMXGZd6+JH/V6F+S5bUDlcCZ6is+KqLXSrcv7innYEUzt63OxRFhxRkVEdKs8O4eD7/dWsxNK7K5yldFBrtmSSYP3nYeK/ISOVDeTHVQP8n+siaOVLXQ6fYEZtDfvta7ZM27x2sDqxQAHChvJicpOrD+2ez0OGIirb0qDf/Io1lpseQnR3Owopm3j9fyD7/fw2v7KwP7uXvOLA+z5VgNZY0dA6603NHVw6GKZnaVNHLjQ1t7jWb6xh/2cuevt/e6J01wc9JAlcYhX2Ls7jF88fGdePpc5I0xVDV3MjstFptFKG3wLocz2D1nNh+pIc5h455LvMO7x3pC5rD/m4wxFcaYnb7vW4BDwIw+u10MvCAiDgARuQt4YIBjbQEGulXc+cBxY0yRMaYLeArYaIzZZ4z5WJ+vavG6H3jVH5s6ez+4fjE/vHHp8DuGUVSktVfT0VDykmLIGWVTWzBntDdpvH2shk88tJXSho5ApQPwlctm86/XLSRliIpmdlos752oo63TzRc+0vvzVE5iNMerW6hodgVGrq3KT2Lf96/kf25e2muejL+DPicpijnp3qSxcVkW939iCfvLmslMcPD7u9dit3mrRX8/CHhHlfnnh8wIrjR8zVNVzd67PC7MiufG87wVXXq8PaTmqbKGDtwew6qCgf82EVYLVy3OZE1hEiX17YHO+cToCF7dXxkYKv3WkWpSYiNZmZdIapydd47VcqKmldm+BAnev+vSbO+y/jmJUSTH2ntVAP6LZobTwYKseA5WNAc61Y8GjWL77osHuPqBt+l09wQmh/qTazB/H8slc1M5WdsWqA46unooqmmlpqWThzadubtjTUtX0Pf9k8bhymZE4JtXzeNwZUu/YdMN7d10uT1kOKNIjIkMDCBobO/ut5S9MYbNR2u4cPaZgRL+wQJjZUQfwUQkH1gO9Lo7jzHmGeA14CkRuQ24E7h5BIeeAQTXXqX0T0zBvgxcDtwoIvcMEuu1IvLzpqamgR5W08B/fHwx/3Pz2SfBhKhIalo7ues3O2ho6+K+GxYHOsoBNi6bwS3n5w55jAtnpzAjIYqn7l7DirzEXo/lJEWzr6wJYyA/ZegkVxhUacxKi+Pxv13N/Z9Ywo0rsnnq7jX84fPryApKFNmJ3u9X5CWyKj8pMIzV3zwV74ig2eXmZ28d54L736S6pZPvXbsQi290W3q8I9BZ//Lecn705yOBYxtj+O4L+9l9upFi34U1b5iJoAUpsfR4DDuKGxCBFXneZisR76g9j4HClFhEhAtmpfDu8VpO1bVx4exUYnzL2OQmR3H72nyeuWctybF2kmMje42e8q+Blh7vYH5GPMV17bzpS5b+dcmqm108s+M0RTVt/H77abrc3hFcAw3T9V+EL53nW83Zd3z/gIxMp4NH3i4KVGTBtxkYMGlUtJCfHMNH53vfQ9t8Cc3Pf/yMeAfJMZG9Etnp+t6V0NGqViqbXVw8JzUwgGWsR4yFnDREJBb4I/BVY0y/M2uM+SHgAh4ErjPGDD3Ntc/hB9g2aMOcMeYBY8wKY8w9xpiHBtnnJWPM3U7n0M0YaurKSYoe1ZyTvpxREXS5PXg88PjfruZTq3OH7bPp6451+bx776WBPpBguUneiyUwbLz+6iLfN49k/ayUQB/UmsLkwNwLP0eElc9fMpOvXzE30P4f57ARa/d2JsdHRdDY3sVP3jjG2pkpvPm1Szg/qFpYXZDE/rJm3jxcxbef3cdDm4sCQ2Qrm138Zmsxf/ywlBLfhXW4+At8SXFbUT3JMfZAZ/vstFgu811E/dXU+lkp1LV14er2MDMthjkZcb7zFUNUpJUVed44k2Ps1PZpnoqKsBLvsLEgy9uM5V+Q8li1999fbz2F22OwWoSf/NU7kTApJjJwJ8pg/mVXVhckB44PZ+5a+b1rF9Lp9gQW1fQnjagI66CVxryMOHKSosh0Onj/ZO/Gl6pApWQnKSaS4G6Wvk1Um31L1180J5WshCgirMKpiZA0RCQCb8J43Bjz7CD7XAgsAp4DvjfCOEqB4DGa2cDEW95RTUv+WeGfXR/akioDGSrJ5CafqQyGmyNzXm4ij9y+ksvmpQ25X7BvbpjH2pnJgaSR5TzzenF2G909hi63h699dE5g6KvfZ9cXkBpn5+9++yHNLjddQUNiT1R7E8XhymaK69qx2yykxQ096KDA14RS2ewiPd4eSCIr8hID65v5k8YFs1KCnhfD3HR/0ugdY0psZL/mqQynAxEJJA2A83ITKKlvp6Gti8e3lXDFgnRWFyRR19bFjIQo1s5M5lDlAEmjvo14h41ZabFYLRKoBA5VNBNrt3GZbyLq/jLvc+tau7BahJlpMb2qDoC2TjfF9e3My4hHRFhdkMS2ovpeHfD+5q/sxGiSfH1+KbHef/1zaMDbJ/PMjlIWZMaT6YzCahGyE6P73czsXAtl9JQAvwQOGWN+NMg+y4FHgI3AZ4EkEfnBCOLYDswWkQIRiQRuAV4cwfOVGjNrZyZz5cL0fn0R54r/IhjvsAVmoA9GRPjogvRRde4v9lU5WUHDkeN9Q55npsYEkkqwGLuNf/zoHLp7TODxo1Xei1pRrW8obEULp+rayU2KDjRrDSYxOiIwzDo93hHo+F+em8g633n2VxwZTkegEpmZGsv1y2dw04rswAXULzk2kvq2Lk7UtNLQ1kVlkyswXDUj3kFidARWi3Dr+bkY411NoLG9mzvW5nPlwgwAzstLZEFmPKfrOwKTHU/Wtnkv8r712KwWITXWHlhj7ZBvJFeE1cL8zPhAv0htaydJMZGkxzn6VRpHqlowxnvrAfDOD6pt7aQoqLP/neO1vtn83uYp8H5YiLXbeiWNZz4s5Vh1K39/2ezAtrzk6EBlNFZCeeetBz4NXCoiu31fV/fZJxq4yRhzwhjjAe4A+g0qFpEnga3AXBEpFZHPARhj3MCXgNfxdrQ/bYw50Pf5SoXDnPQ4Hv70ypDnlIyUv3rxNzmNlSynw9eBfmago3/Ow8fPyx60Grp5ZQ4//uQyHrl9JQDHfU08/tnoLZ1utp+qD2lSqIgEfs/0eO86ZP/58cVctzSLGLuNhz+9MjBCDOCy+WmkxtlJi7OzpjCZ/7ppab84k2O8kykv8w3nrWxykemrpkSElflJrMpPDNyx8tdbi8lyOlhTmMyVCzOItFlYPzOZBb4hrocrWuh093DtT9/hvlcOUVLfHqjAMpyOwHpchytaAsNiF2U5A0N7a1s7SYn1Tk7tO3rKP6TY/zz/bPltRd4mqk53D9uK6rnQNwIxKcab/GalxZKdGMVp3wCCtk43P/rLUVbkJXLlwjP9a/415M5mvbDhDDsj3BjzDgP3OQTv826fn7vxVh5997t1iGO8ArwyXDxKTTWpsXYcEZZ+zS7nmojw0pcu6DUPZ0l2Akuzndzom/8yEKtFuH75mXklZyqNNuw2C51uD00d3eQmhZb0ClNi2HO6kbQ4BxaLDDmI4GsfncvfXTRzyOa98wuSmJ8ZT5zdxuajNVhEAnNMAH78yWV4jCHKt2xKl9vDxuUzAsOy3/nGR0iJtQfmo+wva8Ii3klzL+0pp72rh2t8w4gz4h0cr2mltKGDlk534OK/eIaT375fzKm6Nmpau0iJjSQ1zjuqq8fXdwLeprxYuy0weq0gJYaMeAdvH6vhU6tz2VncSEd3DxfMTgUgyVdVzUyN5Vh1a+D2xf/y0gFqWzt5+NMrep2b3OQYWjvd1Ld1kXyW85MGMylnhCs1lYgIP7h+ca9hvGMlITqyV9KYlRbLC1+6oNdFdihz0mMDK/aeqG7lojmpgceGG/kV2C/ZX2kM/5qRNkugXX8wi2Y4efUrF/LvNyzCY7x3acyIP3PBjLHbiHNEYLNaAv0l/ol/4J1XY7EI6fF2cpKieOd4LdtPeZdeaXa5cXtMIOYMp4OqJhcHfKsY+5uZ/POG9pV5lzRJ9VUaHtN7efPDFS3MzYgLNOOJCJfNT2Pz0Rpc3T28c7wGq0VYU+itQGb4mhLnZ8aTmxRNaUMHL+8t5+kdpXzhkpmcl9t7JJ6/T6y4fuyaqDRpKDUB3Lgie8CRVRPN7PQ4373VuylvcrFkhjNQIYVaKRWknmmeOtex+ZuYBloOH7xDn9cWJg+4lpaIcNm8dN49XsuWozUUpMQE+k+Cm6daOt1sPlpDpM0S6GifnR5LpM3C/rImals7SY6NDKxE4O8MN8ZwyDdyKtjlC9Jp7+pha1Edbx2u4bzcBOJ8o+AumZPG819cz4KseHISo+jo7uGrT+1meW4CX718Tr/fwd9EeKp27DrDNWkopUI2Oy2WLrcnsGhiYWps4CIY6vDmC2alcO3SLFbmhTZJcySuX+6dCZ8ZNEIs2D9fs4An7lo96PMvn59Op9vD1qI61hQm8bEl3uP5Kw3/TPrX9leyNNsZmEQZYbUwPyOO1w9U4er2BPo04MwyLOVNLlpcbuZlxvd6zbWFyURHWvl/z+/nYEUzNyw/01RosQjLcrwfJvznd3G2k8c+c35gBehgeckxxDtsvD3M7ZHPhq5yq5QK2WzfJ3T/vc0LU2NYkZfI1qK6XrPPh5IUE8lPfTfmOtduX5tPYnTkgCPB/IbrH4mz22jpdLMyL4lL56VxXl5ioHLxN6k1tHf3W5ngHz46h7t/8yEAKbF2cpKiibAKX3t6D9+4ci7Jvqplfp9KwxFh5cLZKbx+oIqL5qQOukL0ulnJ3HfDYjYu8w4aGEiE1cJVizJ5eW85HV09REWe+9V9tdJQSoXMv1jgpiM1iHg7cu+8oIA3v3YJkbbwX04cEdbACsKjEWmzcNFcbz/N+QVJJMZEBu4JA71XTF7ZZ2b/JXPTePQzqyhIiWHRDCfp8Q6euGsNOUlRfPu5fWzxVWdzM/o3jX1yVQ5z0+P4rxuXDDps2W6z8qnVuYMmDL/rlmXR1tXDW74bYJ1rWmkopULmiLDy+lcv4vfbS+h0ewKd6qnDTOqbTD5/8UxmpsQElmAJFtx533c5GIALZqfw1tcvCfy8Kj+JH928jI/89yYe31ZCTlJUoL8i2KXz0rl0Xnq/7aOxpjCZ1Dg7L+4uD9yC4FzSpKGUGpGoSCufWV8Q7jDGzKIZzkFXUXZEWEmMjiA1zj7sREy/gpQYLpydwtvHapmXET/8E86S1SJcsziTp7aXjEkTlSYNpZQagasXZwbuIR+q21bn8fax2n79GWPl7y4u5O6LCsekT0OThlJKjcC/37B4xM+5fH4aX7hkJh8/b/BJlOfSYKPHzgVNGkopNcZsVgvf2DBv+B0ngfAPd1BKKTVpaNJQSikVMk0aSimlQqZJQymlVMg0aSillAqZJg2llFIh06ShlFIqZJo0lFJKhUzG8l6yE4GI1DDA/cpDlAKM3cL0o6dxjdxEjU3jGhmNa+RGE1ueMSZ1oAemfNI4GyKywxizMtxx9KVxjdxEjU3jGhmNa+TOdWzaPKWUUipkmjSUUkqFTJPG0H4e7gAGoXGN3ESNTeMaGY1r5M5pbNqnoZRSKmRaaSillAqZJg2llFIh06QxABHZICJHROS4iNwbxjhyROQtETkkIgdE5Cu+7d8XkTIR2e37ujpM8Z0SkX2+GHb4tiWJyF9E5Jjv38Rxjmlu0HnZLSLNIvLVcJwzEXlURKpFZH/QtgHPj3g94HvP7RWR88IQ23+JyGHf6z8nIgm+7fki0hF07h4a57gG/duJyLd85+yIiFw5znH9PiimUyKy27d9PM/XYNeIsXufGWP0K+gLsAIngEIgEtgDLAhTLJnAeb7v44CjwALg+8DXJ8C5OgWk9Nn2Q+Be3/f3AveH+W9ZCeSF45wBFwHnAfuHOz/A1cCrgABrgG1hiO0KwOb7/v6g2PKD9wtDXAP+7Xz/F/YAdqDA9//WOl5x9Xn8f4DvhuF8DXaNGLP3mVYa/Z0PHDfGFBljuoCngI3hCMQYU2GM2en7vgU4BMwIRywjsBH4te/7XwPXhzGWy4ATxpjRrghwVowxW4D6PpsHOz8bgd8Yr/eBBBHJHM/YjDF/Nsa4fT++D4zPDa2HiWsIG4GnjDGdxpiTwHG8/3/HNS4REeBm4MmxeO2hDHGNGLP3mSaN/mYAp4N+LmUCXKhFJB9YDmzzbfqSr7x8dLybgIIY4M8i8qGI3O3blm6MqQDvGxpIC1NsALfQ+z/yRDhng52fifa+uxPvJ1K/AhHZJSKbReTCMMQz0N9uopyzC4EqY8yxoG3jfr76XCPG7H2mSaM/GWBbWMcli0gs8Efgq8aYZuBBYCawDKjAWxqHw3pjzHnAVcAXReSiMMXRj4hEAtcBz/g2TZRzNpgJ874TkX8G3MDjvk0VQK4xZjnwj8ATIhI/jiEN9rebKOfsVnp/OBn38zXANWLQXQfYNqJzpkmjv1IgJ+jnbKA8TLEgIhF43wyPG2OeBTDGVBljeowxHuARxqgkH44xptz3bzXwnC+OKn+56/u3Ohyx4U1kO40xVb4YJ8Q5Y/DzMyHedyJyB/Ax4DbjawT3Nf/U+b7/EG/fwZzximmIv13Yz5mI2ICPA7/3bxvv8zXQNYIxfJ9p0uhvOzBbRAp8n1ZvAV4MRyC+ttJfAoeMMT8K2h7cBnkDsL/vc8chthgRifN/j7cTdT/ec3WHb7c7gBfGOzafXp/+JsI58xns/LwI3O4b3bIGaPI3L4wXEdkAfBO4zhjTHrQ9VUSsvu8LgdlA0TjGNdjf7kXgFhGxi0iBL64Pxisun8uBw8aYUv+G8Txfg10jGMv32Xj08E+2L7wjDI7i/YTwz2GM4wK8peNeYLfv62rgt8A+3/YXgcwwxFaId+TKHuCA/zwBycAbwDHfv0lhiC0aqAOcQdvG/ZzhTVoVQDfeT3ifG+z84G02+JnvPbcPWBmG2I7jbe/2v9ce8u37Cd/feA+wE7h2nOMa9G8H/LPvnB0BrhrPuHzbHwPu6bPveJ6vwa4RY/Y+02VElFJKhUybp5RSSoVMk4ZSSqmQadJQSikVMk0aSimlQqZJQymlVMg0aSillAqZJg2llFIh+/8BrNw3rpjtf8sAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pd.Series(1-np.array(acc)).plot(logy=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x13c3860b8>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAEICAYAAABLdt/UAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXyU1aH/8c+Z7IFAAglr0IAgsio0gAuyWAXcUHttta1WW61Wa3tt721rbbW22l+9te11uXbxXq22Wlur1r2K1SCiVMuuQMK+BAJZIIHss5zfH88MBAwwkHmeWfJ9v155zcwzzzznPAn5cnKe85xjrLWIiEjq8MW7AiIiElsKdhGRFKNgFxFJMQp2EZEUo2AXEUkxCnYRkRSTHu8KABQWFtqSkpJ4V0NEJKksWbKk1lpbdOj2hAj2kpISFi9eHO9qiIgkFWPMls62x7wrxhhzqTHmf40xLxpjZsX6+CIicmRRBbsx5jFjTLUx5uNDts8xxlQYY9YbY24DsNa+YK39KnAtcEXMaywiIkcUbYv9cWBOxw3GmDTgYeB8YDTweWPM6A67/DD8voiIeCiqPnZr7QJjTMkhmycD6621GwGMMX8GLjHGrAHuBf5urV16uGMaY24AbgA44YQTPvG+3++nsrKS1tbWaKooQHZ2NsXFxWRkZMS7KiISR125eDoY2NbhdSUwBfgGcC7Q2xgz3Fr7284+bK19BHgEoLS09BMzkVVWVpKXl0dJSQnGmC5Us3uw1lJXV0dlZSVDhw6Nd3VEJI66Euydpa211j4IPNiF4wLQ2tqqUD8Gxhj69u1LTU1NvKsiInHWlVExlcCQDq+LgR1dq87BFOrHRt8vEYGutdj/BYwwxgwFtgNXAl+ISa1ERFLUhppGNlXvo2VnBYN3vsVJl/2Q3rlZMS0j2uGOTwOLgJHGmEpjzHXW2gBwC/AGsAZ4xlq76lgKN8ZcbIx5pKGh4Vjr7YnNmzczduxYV449f/58LrroIgBeeukl7r33XlfKEZHE8fKyLfzp/u8y+C/ncfG7l3Dq2ofYu+GDmJcT7aiYzx9m+2vAa8dbuLX2ZeDl0tLSrx7vMVLB3LlzmTt3bryrISIu+ue6nWT87XruyPiQth6D2Fv6YwIj5jBw4MkxL0uTgB1FIBDgmmuuYfz48Vx++eU0Nzfzk5/8hEmTJjF27FhuuOEGIssLPvjgg4wePZrx48dz5ZVXAtDU1MRXvvIVJk2axIQJE3jxxRc/Ucbjjz/OLbfcAsC1117LN7/5Tc4880yGDRvGs88+u3+/++67j0mTJjF+/Hh+9KMfeXD2IhILFdvraHzqaub4PqTlnHvI+s4aes28lT7Fp5CeFvsYToi5Yo7mxy+vYvWOvTE95uhBvfjRxWOOul9FRQWPPvooZ511Fl/5ylf49a9/zS233MKdd94JwNVXX80rr7zCxRdfzL333sumTZvIysqivr4egJ/+9Kecc845PPbYY9TX1zN58mTOPffcI5ZZVVXFwoULKS8vZ+7cuVx++eXMmzePdevW8eGHH2KtZe7cuSxYsIBp06Z1/ZshIjH1dvkubn/+Y/zBED0zfdzRci/n8iH10+8hf9o3XC9fLfajGDJkCGeddRYAV111FQsXLqSsrIwpU6Ywbtw43n77bVatci4tjB8/ni9+8Ys8+eSTpKc7/2fOmzePe++9l9NOO40ZM2bQ2trK1q1bj1jmpZdeis/nY/To0ezatWv/cebNm8eECROYOHEi5eXlrFu3zsUzF5Hj8fv3NnH9E4sp6JHJnLEDuLh/LefyITWT/pP8me6HOiRJiz2alrVbDh1CaIzh5ptvZvHixQwZMoS77rpr/92xr776KgsWLOCll17i7rvvZtWqVVhree655xg5cuRBx4kEdmeysg5cIY9081hr+f73v8+NN94Yq1MTkRjyB0Pc88pqnli0hVmj+3P/laeRm5kOS5bCZig63btBg3FtsSf6qBiArVu3smjRIgCefvpppk6dCkBhYSGNjY37+8BDoRDbtm1j5syZ/PznP6e+vp7GxkZmz57NQw89tD+gly1bdlz1mD17No899hiNjY0AbN++nerq6q6enojEwMfbG5j7P+/xxKItfPXsofzmqk85oQ6wcyVk5kGBd3eEx7XFngyjYkaNGsUTTzzBjTfeyIgRI7jpppvYs2cP48aNo6SkhEmTJgEQDAa56qqraGhowFrLt771LfLz87njjju49dZbGT9+PNZaSkpKeOWVV465HrNmzWLNmjWcccYZAPTs2ZMnn3ySfv36xfR8RSR6wZDl/n+s5dfzN9CnRyaPXP0pZo0ZcPBOVSthwDjwedeONpGWZDyVlpbaQxfaWLNmDaNGjYpTjZKXvm8i3nl04SbufmU1n5k4mDsvGk1+bubBO4SC8LNimHgNnB/7e1WMMUustaWHbk+KPnYRkURT39zOg2+t4+wRhfzqc6d1vlPdevA3w8DxntZNo2JERI7DA2+tY1+rnx9ceIS/kKtWOo8DFOwiIgltY00jf1y0hSsmncApA3odfsedKyAtC4pGHn4fFyjYRUSO0c/+Xk5Wuo9vn3eU6QCqVkL/0ZDm7eI3Gu4oInIMFm2o483Vu7h55nCK8o4wK6O1ULXC824YiHOwW2tfttbe0Lt373hWQ0QkKqGQ5Z5XVzM4P4frph5lXHrDNmit9/zCKagrRkQkas8v286qHXv57pyRZGekHXnn/RdOT3W/YodQsLsgEAjEuwoiEmPN7QHue6OcU4fkM/fUQUf/wM6VYHzQ3/spURTsR/CrX/2KsWPHMnbsWO6///5PLLzxi1/8grvuuguAGTNmcPvttzN9+nQeeOCBONVYRNzyyIKN7Nrbxh0XjopuGcqqlVB4MmTmul+5QyTHDUp/vw12fhTbYw4Yd8Q7wZYsWcLvf/97PvjgA6y1TJkyhenTpx/xkPX19bzzzjuxraeIxN3OhlZ+985GLhw3kNKSPtF9qGoFlEx1t2KHkRzBHgcLFy7ksssuo0ePHgB85jOf4d133z3iZ6644govqiYiHvvFvAqCIcv35pwS3QeaamHfjrhcOIVkCXYX5lg4ms7m0KmvrycUCu1/HZmuNyLyn4CIpI6Ptzfw3NJKbjh7GCf0jbJbZVd4+ef+7qyZfDQax34Y06ZN44UXXqC5uZmmpib+9re/cf7551NdXU1dXR1tbW3HNUujiCQPay0/fXUNBbmZ3DxzePQfrF3rPBZF2cKPMU3bexgTJ07k2muvZfLkyQBcf/31TJo0iTvvvJMpU6YwdOhQTjklPj80EfHGP9ZUs2hjHXdfMobeOcdw92hNOWT1grwBR9/XBZq2N8Xo+yYSG+2BELPvX4DPwBu3Tju2RacfvwgCrXD9P9yrIIeftlfDHUVEOvHUB1vYVNvEDy4cdWyhDk5XTKG3E391pGAXETlEQ7OfB95ax9ThhcwceYyrlLXsgcZdUHSUCcJclNDBngjdRMlE3y+R2Hjo7XU0tDhzrUd1M1JHNfG9cAoJHOzZ2dnU1dUprKJkraWuro7s7Ox4V0UkqW2ubeKJRZu5onQIowYeYa71w6mtcB4L49diT9hx7MXFxVRWVlJTUxPvqiSN7OxsiouL410NkaR279/LyUjz8e1ZxxnMNRWQng35J8S2YscgYYM9IyODoUOPMi2miEgMfbCxjtdX7eQ/zjuZfnnH+ddvTQUUjgDfUWZ/dJFuUBIRITLX+hoG9s7m+rOHHf+BairiOiIGtNCGiAgAL67YzkfbG/junJHkZB5na7u9CRq2xvXCKSTwxVMREa+0tAf5+esVjC/uzSWnDj7+A9Wucx7jONQRFOwiIvzfuxupamjlhxeOxuc7xuGNHdVERsR0464YEZF4q97bym/e2cD5YwcweWiUc60fTm0F+NKhTxf66GNAwS4i3drj72+mPRDitvNj0C9eU+GEenpm14/VBQp2EenW3i6vZvLQPpzYNwbrKdRUQFF8u2FAwS4i3diO+hbKd+479vlgOhNoh90b496/Dgp2EenG5lc4d7bPPKWo6wfbvQFsUC12EZF4eru8muKCHE4q6tn1g9Wtdx4LR3T9WF2kYBeRbqktEOS99bXMHNnv2Gdw7Ewk2Puc1PVjdZGmFBCRbunDTbtp8Qdj0w0DTrD37A/ZxzEjZIxpSgER6ZbKymvITPdxxrDC2BywbgP0PYYFr12krhgR6ZbmV1RzxrC+xz8vzKHqNsT9xqQIBbuIdDuba5vYWNvEzJEx6oZpbYCmarXYRUTiZX5FNQAzYjF+HZzWOijYRUTipayihmGFPSgpjMHdpuDcmATQN/4jYkDBLiLdTEt7kEUb65h5Soxa6xAe6migIDFWfVOwi0i3smhjLe2BUGymEYioWw/5QyAjMRaTV7CLSLdSVl5DbmYak4YWxO6gdesTpn8dFOwi0o1YaymrqOas4YVkpcdomKO1ULcxIe44jVCwi0i3saGmkco9LbHthmmqhbYGtdhFROKhrNyZzXFGrMavw4E5YhTsIiLeK6uo5pQBeQzKz4ndQfcHu7piREQ8ta/Vz782747dTUkRuzeALwN6D4ntcbtAwS4i3cJ76+vwB23sphGIqFsPfYZCWnpsj9sFmrZXRLqF+RXV5GWnM/HEGA5zhISa1TFC0/aKSMqLDHOcNqKIjLQYxl4o5EwnkED966CuGBHpBlZX7WXX3rbYjoYB2LsdAq0JNYYdFOwi0g1EFq2eHutgr13rPKorRkTEW2Xl1Ywb3Jt+eTGey2XnSudxwNjYHreLFOwiktLqm9tZunVP7EfDAOxYDvknQk6ML8h2kYJdRFLagnW1hCzMiOU0vRFVy2HQabE/bhcp2EUkpc0vr6YgN4NTi/Nje+CWPbBnMwxUsIuIeCYUssxfW8P0k4tI85nYHrwq3L8+8NTYHjcGFOwikrJWbm9gd1N7bFdLiqha4TyqxS4i4p2y8mp8BqaNcOHCadVyZ36YHn1jf+wuUrCLSMqaX1HNhBMKKOiRGfuD71iekN0woGAXkRRVs6+NFZUN7gxzbN3rzOqYgCNiQMEuIilqwdrIohou9K9HbkxKwP51ULCLSIoqq6imX14WYwb1iv3Bdyx3HtUVIyLijUAwxIK1NcwYWYQxMR7mCM6ImLxB0NOFvwZiQMEuIiln2bZ69rYGYrtodUcJesdphIJdRFJOWXk16T7DWSMKY3/wtkaoXZew/eugYBeRFFRWUUNpSQG9sjNif/CdHwE2YfvXQcEuIimmqqGFNVV73e2GAXXFHI7WPBWRWIssquHKNALgXDjtOQDyBrhz/BjQmqciklLKyqsZnJ/DiH493Skgge84jVBXjIikjLZAkPfW17o3zLG9GWorErobBhTsIpJCFm/eQ1N70L3+9V0fgw2pxS4i4pWy8moy03ycOdylGRf333GqFruIiCfKKqqZMqwPuZnp7hRQtQJ6FEGvQe4cP0YU7CKSErbWNbOhpsm9bhhwhjoOPBXc6L+PIQW7iKSE+WurAReHOfpboHpNwnfDgIJdRFJEWXk1JX1zGVrYw50Cdq0GG0z4C6egYBeRFNDqD/L+hjp35l6PqFrmPCb4UEdQsItICli0sY62QIhz3OqGAefCaU4fZ53TBKdgF5GkN7+8mpyMNCYP7eNeITuS48IpKNhFJMlZaymrqOGs4X3Jzkhzp5BAm3PhNAm6YUDBLiJJbmNtE1t3N7vbv169GkL+pLhwCgp2EUlyZeXOMMcZI4vcKyRJ7jiNULCLSFKbX1HDyf17UlyQ614hVSsguzcUlLhXRgwp2EUkaTW1BfhgU527d5tC0txxGqFgF5Gk9d76WvxB627/eqAddq1Kmm4YULCLSBIrq6imZ1Y6pSUF7hVSUw7B9qQZEQMKdhFJUtZaysprOHtEIRlpLkZZVXJdOAUFu4gkqfKd+9i5t9X9/vUdyyGrFxQMdbecGFKwi0hSKqtwhjlOd3OYIzgjYgaMB1/yxGXy1FREpIP55TWMGdSL/r2y3SskGHCWw0ui/nVQsItIEmpo9rNk6x73u2FqKyDQmlT966BgF5Ek9O76GoIhy8xTXO6G2X/HaXJMJRChYBeRpFNWXkN+bganDXFxmCM4I2Iye0Lf4e6WE2MKdhFJKqGQ5Z211UwbUUSaz+U7QatWwIBxSXXhFBTsIpJkPt7RQG1ju/vdMKEg7Pwo6frXQcEuIkmmrLwGY2DaCJeDvXYd+JuTbkQMKNhFJMmUVVRzanE+fXtmuVtQVXJeOAUFu4gkkbrGNlZU1ru7tmnEjuWQngOFJ7tfVowp2EUkaSxYV4O1uD9+HTpcOHVpuT0XxTzYjTHDjDGPGmOejfWxRaR7KyuvobBnFmMG9XK3oFAIdq5Myv51iDLYjTGPGWOqjTEfH7J9jjGmwhiz3hhzG4C1dqO19jo3Kisi3VcwZHlnbQ0zRhbhc3uYY916aG9MyhExEH2L/XFgTscNxpg04GHgfGA08HljzOiY1k5EJGz5tj00tPi964aBpLxwClEGu7V2AbD7kM2TgfXhFno78GfgkmgLNsbcYIxZbIxZXFNTE3WFRaR7KiuvIc1nmDqi0P3CqpZDejYUneJ+WS7oSh/7YGBbh9eVwGBjTF9jzG+BCcaY7x/uw9baR6y1pdba0qIil8ejikjSe7u8mk+dWEDvnAz3C6taAf3HQlq6+2W5oCu17qyTy1pr64CvdeG4IiIH2dnQyuqqvXxvjgct6FDICfZxn3W/LJd0pcVeCQzp8LoY2NG16oiIfNI7a51FNVyfRgBgzyZo25u0I2Kga8H+L2CEMWaoMSYTuBJ4KTbVEhE5oKy8hoG9sxnZP8/9wpL4jtOIaIc7Pg0sAkYaYyqNMddZawPALcAbwBrgGWvtqmMp3BhzsTHmkYaGhmOtt4h0E+2BEAvX1zJjZD+McXmYIzh3nKZlQtEo98tySVR97Nbazx9m+2vAa8dbuLX2ZeDl0tLSrx7vMUQktS3espvGtgAz3V7bNKJqBfQfA+mZ3pTnAk0pICIJbX5FDRlphrOGezDM0Von2JO4GwYU7CKS4MrKq5kytC89sjwYerhnM7TWJ+0dpxEKdhFJWNt2N7OuupEZXnbDgFrsIiJumb/WuSt9phfT9IIzIsaX4fSxJ7G4BrtGxYjIkcwvr+aEPrkMK+zhTYE7lkO/UZDu8iIeLotrsFtrX7bW3tC7d+94VkNEElCrP8h7G2qZObLIm2GOKXLhFNQVIyIJ6oNNu2n1h5jhVTdMwzZo2Z3Ud5xGKNhFJCGVlVeTneHjjGF9vSlw/4VTBbuIiCvmV1Rz5kmFZGd4tDTdjuVg0pL+wiko2EUkAW2qbWJzXbN3d5uCMyKm3yjIyPGuTJco2EUk4ZSVO7M5zvBitSRwLpzuWJ4SF05Bwx1FJAGVVVQzvF9PhvTJ9abAvTuguTYl+tdBwx1FJME0tQX4YONuj7thwhdOU2BEDKgrRkQSzPsb6mgPhrxZtDqiajkYn7McXgpQsItIQimrqKZHZhqlJX28K3THcigcCZkedf24TMEuIgnDWsv88mqmjigkM93DeEqRO04jFOwikjDW7mpkR0Ort90w+3ZC486U6V8HBbuIJJCyCo+HOYLTDQMpMyIGFOwikkDKyqsZNbAXA3pne1fo2tedC6cDUuPCKWgcu4gkiL2tfhZv2ePtMMeN78CS38Okr0JWnnflukzj2EUkISxcV0swZL1bVKO1AV78OvQdDufe5U2ZHvFgEUERkaMrK6+mV3Y6E4bke1Pg67fD3u1w3ZspM8wxQn3sIhJ3e1v9vF1ezbSTi0hP8yCWyl+D5U/C1G9Dcan75XlMwS4icWWt5bt/XUl9i58vnzXU/QKbauHlb8KAcTD9e+6XFwfqihGRuHrsvc28vmonP7hgFJ86scDdwqyFV251+te/9BKkZ7pbXpyoxS4icbNky25+9toaZo3uz/Vne9BaX/kMrHkZZv4A+o92v7w4UbCLSFzUNbbx9aeWMSg/h/s+e6r7C1Y3VMJr34Ehp8OZ33C3rDhTV4yIeC4Ystz6l+Xsbm7n+ZvOpHdOhrsFhkLO0MZQAC77Dfg8Wm4vTnSDkoh47qG31/Huulp+MncMYwd7cB/L4kdh43yYfQ/0GeZ+eXGmG5RExFML1tbwwFvr+MzEwVwxaYj7BdZtgHl3wPBz4VNfdr+8BKA+dhHxTFVDC7f+ZTkn98vjnkvHut+vHgzA326E9CyY+xC4XV6CUB+7iHjCHwxxy5+W0eYP8uurJpKb6UH8vP8AVP4L/u1R6DXI/fIShIJdRDzxX38vZ8mWPTz0+QmcVNTT/QJ3fgRlP4PRl8LYf3O/vASirhgRcd3rH1fxfws3ce2ZJVx8qgct50AbPH8j5PaBC3/VbbpgItRiFxFXba5t4jt/XcmpQ/K5/YJR3hQ6/2dQvQq+8Az06OtNmQlELXYRcU2rP8hNTy0lLc3w8BcmeLOO6dYP4L0HYOKX4OTZ7peXgNRiFxHX/OjFVayp2svvvzyJ4gIPpsZta3RGwfQuhtn/z/3yEpSCXURc8dfF2/jL4m3cMnO4d4tTv3kn7NkM176SUisiHSt1xYhIzK2p2ssdL37MmSf15VvnnexNoev/4dxhesbXoWSqN2UmKE0pICIxta/Vz81PLaVXdgYPXDmBNJ8HI1Ja9sCLt0DhSDjnDvfLS3CaUkBEYsZay23PfcTW3c089PkJFOVleVPwa9+Fphr4zO8gI9ubMhOYumJEJGYef38zr35UxXdmj2TKMI+GGa56AT56BqZ9BwZN8KbMBKdgF5GYWLp1Dz99dQ3njurPDWd7NIPivl3wyrecQD/7P7wpMwko2EWky3Y3tXPLU0sZmJ/NLz97Kj4v+tWtddYubW+Cy34HaS7P6Z5ENNxRRLokFF40o7axneduOpPeuR4F7LInYe3rMPtnUDTSmzKThFrsItIlD5etZ8HaGn40dzTjij0aCLFnC7z+fSg5G6Z8zZsyk4iCXUSO28J1tfzqH2u59LRBfGHyCd4UGgrBCzc7zy95GHyKsUOpK0ZEjsvOhlb+/c/LGF7Uk59eNs79RTMiPvgtbFkIc/8HCk70pswko//qROSY+YMhvvH0Ulr8QX5z1UR6ZHnURqypgH/cBSfPgQlXeVNmElKLXUSO2X1vVPCvzXt44MrTGN7PozlZgn5ngq/MHnDxg91ujvVjoWAXkWPyxqqdPLJgI1effiKXnDbYu4Lf/SXsWAaffQLy+ntXbhJSV4yIRG1LXRP/+dcVjC/uzQ8v8mjRDIDtS2HBfTDuczDmUu/KTVIKdhGJSqs/yM1PLcVnDA9/YSJZ6WneFOxvgb99DXr0gwt+7k2ZSU5dMSISlR+/vJpVO/by6DWlDOnjwaIZEW/fA7UVcNVzkFPgXblJTNP2ishRPbekkqc/3MpNM07i06M87N/evBAWPQyl18Hwc70rN8lp2l4ROaKKnfv4wQsfMWVoH/7Dq0UzANr2wQs3QUEJzLrbu3JTgLpiROSwGtsC3PTUEvKyM3joCxNIT/OwLfjG7dBQCV/+uzPEUaKmi6ci0ilrLd97biWba5t48MoJ9MvzcAGLtW/A0j/Amd+EE073rtwUoWAXkU79YdEWXl1ZxX/OHskZJ3m0aAZAU52zzF2/MTDzdu/KTSHqihGRT1i+rZ57Xl3Np0/px9emneRdwdbCq9921jC9+nlI92hpvRSjFruIHGRPUztff2op/fKy+eXnPFo0I+Lj52D1CzDjNhgwzrtyU4xa7CKyXyhk+fYzy6nZ18azN51Bfm6md4Xv3eG01osnwVm3elduClKwi3RzoZBl5fYG5q3aybzVu1hf3cjdl4xhfHG+d5XYscxZOCPQHl7mTtHUFfruiXRDbYEgizbU8ebqXby5ehfV+9pI8xkml/Th+qlDuWLSEPcr0doAH/0VljwBO1dCejZc9N/Q18M+/RSlYBfpJva2+ikrr+bN1buYX1FDY1uA3Mw0pp9cxHmj+3POKf3c73qxFir/5YT5qufB3wz9x8IFv4Bxn4UcD/9KSGEKdpEUVtXQsr9V/s+NdfiDlsKemVw0fiCzxvTnzJMKyc7wYDKv5t2w8hlY+gRUr4aMHjDucph4LQyeqLnVY0zBLpJCrLVU7NrHm6t2MW/1Lj7a7szDNKywB1+ZOpRZo/tz2pAC0rwY6WItbHnfCfNVL0CwDQZNgIvud0I9y6MFOrohBbtIkguGLIs37+bN1U6Yb93dDMCEE/L57pyRzBo9gOH9enpXoaZaWP4n587RunWQ1QsmXg0Tr4GB472rRzemYBdJQi3tQRasq+HN1bt4u7ya3U3tZKb5OHN4X742/STOHdWPfr08nAIgFILNC2DJ47DmFQj5YcgUmPprZ2EMzfXiKQW7SJKoa2zjrfJq5q3axcL1NbT6Q+Rlp/PpU/px3ugBTB9ZRE+vFpWO2LcLlj/ptM73bIbsfJj8VZj4Jejn4QpLchAFu0gC21LX5HSxrNrF4i27CVkY1DubK0qHMGvMACYP7UOGlzMuAoSCsOFtp3W+9nUIBaDkbJj5Qxh1MWR4+JeCdErBLpIA9rX6qdzTwvY9LVTuaWbr7hbeW19Lxa59AJwyII9bzhnBrNH9GTOoFyYeo0gatsOyJ2HZH6FhG+QWwuk3O33nhcO9r48cloJdxGXWWva2BKisb6ZyT8tBAV65p4Xt9S00tPgP+kxWuo/ThuRzx0WjmTW6v7dL0XUUDMC6eU7rfP2bYEMwbCbMugdGXgDpHk45IFFTsIt0kbWW+mZ/OLSb2V7fsv95JMT3tQUO+kxuZhrFBTkMzs/hUycWOM8LciguyKW4IIe+PTLj0yqP2LMZlv4Rlj8F+6qg5wCY+m1ndEtBSfzqJVGJa7AbYy4GLh4+XH/GSeKy1lLX1B5uZR/c0o48b24PHvSZnlnpFBfkUFyQw+nD+jI4Pyf8OpfBBTkU5GbEN7jBGcnSVAP7dsDeKifA91XB9iWwocy5aWj4eXDhL2HEbM3fkkSMtTbedaC0tNQuXrw43tWQJGStxR+0tAaCtPqDtPlDtPqDtPpDB28LhLf5w9sCnT0PP4Zft/mD7GsLsKO+hVZ/6KBye2Wn729dd2xpD6RuXv4AAAoYSURBVM7PYUhBLr1y0uMb3G37Dg7rvTs6PA8/Nu5yLnx2ZHxOi3z8FTDhKuhdHJfqS3SMMUustaWHbtd/wd1cMGRp8QdpbgvQ3B6kqT1AS3uQpvYgLe2Rbc7zlvYQQWux1hKylmCI/c9D1jmWDT8PRbaH2P++tZZgh/ettQRDB96LHMN5j3AZzvP2YOgTgdzqD9EWCBI6zraJMZCdnkZ2ho/sjDSy0sOPGWlkp/vIz81kcEEO54zsFw7wA0HeKzsjtj+IaAUDTiB3GtY7YN9O53n7vk9+Nqs39BoIeQOgcHr4efir10DIGwQ9itQyTwH6CSa4QDBEezBEe8D5ags4YdbcHqSpLUiLP+A8hkO5uf3A844B3dQWpNl/4HmLP0hTW4C2QOjolTiEz4DPGOfLd+C5MZDmC283YMKPacY4z8P7poX3PXAM0+GYzufSfB2O4YO8jHSK8rLIDodudoYTyFkdg/mg9w5sz05PIyvDtz/Es8LvZab54tuqDgWdlvUnvvY6X401HcI6HOKN1cAh/5P50g8EdL9RcNI54bAedHBw6yahbiOpg/3DB77A4PolAPt/QQ0H5hPquA0TfsQJlcivswm/edDn9u9vOuwTOfaBY1qclqbzGP51O+i17bD94NcdP8cnjnPw5w6VDvQKf3Umcn7mkIA1kUDFYNLA1yP8vYi8f7TP+AwmPduZXjUj1xmvnJED6TnOY0bOIe/lhl93fC/nMO/lJE9LMdDeIYA7C+Votu0Df9PRy8rpEw7oAc6KQpHneYMOtLJz+4JPi6HJAUnym9S5YJ8R7Ai0YeGgP987feTQ7eEw7dBNcOxt18PzQbglenDLNq1jK9cXfm3o0HI1pJkjfzbN5yPNZ0gPf6X5DOlpPtKMIT0t3HqO4bnsZ0MQaHW+/C3OjH3+Fgi0gL/1wPNg+/Ed35fRyX8CcQ79UOiTwRxsO/rnjM+Z5CqrV/gxD3L7OP3Xkdcd3/vEtp7OOHHd7CPHIamD/YyrfxzT40X6fAORr2AIf9DZ5g+GCIQswdCBbRlpPjLTw1/h51nh556uE5loQsFwyLc68237w48dXwdanH0iX0d7L+g/erluMD6nlXzYAD7MtoxcTUUrcZPUwR5rJtziTfdgeuqU5ktzWpxZHs4oKCL7qWNORCTFKNhFRFKMgl1EJMUo2EVEUoyCXUQkxSjYRURSjIJdRCTFKNhFRFJMQkzba4ypAbYc58cLgdoYVicZ6Jy7B51z99CVcz7RWlt06MaECPauMMYs7mw+4lSmc+4edM7dgxvnrK4YEZEUo2AXEUkxqRDsj8S7AnGgc+4edM7dQ8zPOen72EVE5GCp0GIXEZEOFOwiIikmaYLdGDPHGFNhjFlvjLmtk/ezjDF/Cb//gTGmxPtaxlYU5/xtY8xqY8xKY8xbxpgT41HPWDraOXfY73JjjDXGJP3QuGjO2RjzufDPepUx5k9e1zHWovi3fYIxpswYsyz87/uCeNQzVowxjxljqo0xHx/mfWOMeTD8/VhpjJnYpQIj634m8heQBmwAhgGZwApg9CH73Az8Nvz8SuAv8a63B+c8E8gNP7+pO5xzeL88YAHwT6A03vX24Oc8AlgGFIRf94t3vT0450eAm8LPRwOb413vLp7zNGAi8PFh3r8A+DvOmvKnAx90pbxkabFPBtZbazdaa9uBPwOXHLLPJcAT4efPAp82JqkXnTzqOVtry6y1zeGX/wSKPa5jrEXzcwa4G/g50Opl5VwSzTl/FXjYWrsHwFpb7XEdYy2ac7ZAr/Dz3sAOD+sXc9baBcDuI+xyCfAH6/gnkG+MGXi85SVLsA8GtnV4XRne1uk+1toA0AD09aR27ojmnDu6Dud//GR21HM2xkwAhlhrX/GyYi6K5ud8MnCyMeY9Y8w/jTFzPKudO6I557uAq4wxlcBrwDe8qVrcHOvv+xEly2LWnbW8Dx2nGc0+ySTq8zHGXAWUAtNdrZH7jnjOxhgf8N/AtV5VyAPR/JzTcbpjZuD8VfauMWastbbe5bq5JZpz/jzwuLX2l8aYM4A/hs855H714iKm+ZUsLfZKYEiH18V88k+z/fsYY9Jx/nw70p8+iS6ac8YYcy7wA2CutbbNo7q55WjnnAeMBeYbYzbj9EW+lOQXUKP9t/2itdZvrd0EVOAEfbKK5pyvA54BsNYuArJxJstKVVH9vkcrWYL9X8AIY8xQY0wmzsXRlw7Z5yXgmvDzy4G3bfiqRJI66jmHuyV+hxPqyd7vCkc5Z2ttg7W20FpbYq0twbmuMNdauzg+1Y2JaP5tv4BzoRxjTCFO18xGT2sZW9Gc81bg0wDGmFE4wV7jaS299RLwpfDomNOBBmtt1XEfLd5Xi4/hqvIFwFqcq+k/CG/7Cc4vNjg/+L8C64EPgWHxrrMH5/wPYBewPPz1Urzr7PY5H7LvfJJ8VEyUP2cD/ApYDXwEXBnvOntwzqOB93BGzCwHZsW7zl0836eBKsCP0zq/Dvga8LUOP+OHw9+Pj7r671pTCoiIpJhk6YoREZEoKdhFRFKMgl1EJMUo2EVEUoyCXUQkxSjYRURSjIJdBDDGvGaMyY9j+ZcaY0bHq3xJLQp2SSnGmLTj+Zy19gIb37lXLsW5KUekyxTskjSMMSXGmHJjzBPhxQieNcbkGmM2G2PuNMYsBD5rjJkfmT/GGFMYnlcGY8y1xpjnjTGvG2PWGWN+3uHYm8P7lhhj1hhj/je8qMU8Y0xOeJ9J4XIXGWPuO9yiCR3KejFcVoUx5kcd3vtS+DgrjDF/NMacCcwF7jPGLDfGnOTOd1C6CwW7JJuRwCPW2vHAXpwFVgBarbVTrbV/PsrnTwOuAMYBVxhjhnSyzwic+c/HAPXAv4W3/x7nFvAzgGAUdZ0MfDFc5meNMaXGmDE4k7adY609Ffh3a+37OHOFfMdae5q1dkMUxxY5LAW7JJtt1tr3ws+fBKaGn/8lys+/ZZ3JxFpx5l7pbDnBTdba5eHnS4CScP97XjiEAaJZnu5Na22dtbYFeD5c13OAZ621tQDW2mSegVQSlIJdks2hkxtFXjd12BbgwL/t7EP27zi1cZDO1yTobJ/jWY2rs7qaTraLxJSCXZLNCeGFF8BZjGFhJ/tsBj4Vfn55LAq1zrJ0+8JTqoIz1ezRnGeM6RPuo78UZ7bCt4DPGWP6Ahhj+oT33Ycz37xIlynYJdmsAa4xxqwE+gC/6WSfXwA3GWPeJ7aLM1wHPGKMWYTT8m44yv4LgT/iTDv7nLV2sbV2FfBT4B1jzAqc6XjBWffzO8aYZbp4Kl2laXslaRhjSoBXrLVj41R+T2ttY/j5bcBAa+2/H2bfa3Hm1L7FwyqKAMmz5qlIIrjQGPN9nN+bLaTW2quSQtRiF+kCY8xs4L8O2bzJWntZPOojAgp2EZGUo4unIiIpRsEuIpJiFOwiIilGwS4ikmL+P+JnXR6YWTE6AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "data = pd.DataFrame(pd.Series(x).rename('pruning_pct'))\n",
    "data['baseline'] = 100*(1-np.array(ya))\n",
    "data['our'] = 100*(1-np.array(yb))\n",
    "data = data.set_index('pruning_pct')\n",
    "data.plot(logy=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAQEUlEQVR4nO3df4ylVX3H8fdX1h9ELaDg1OxuOiRuCNhtxUyAxn8moLCKcWkjyRqjS7vN/lFMMNmkLvUPUpUE0yCNVm22ZeNqiEj8kSViQ7fIjWkiP4WywJbuFmmZQiRmER2MNGO//WPO6p3dO3PvzNx57nPnvF/JZO5znnPvc56zdz7PueeeezcyE0lSHV416gZIkppj6EtSRQx9SaqIoS9JFTH0JakiG0bdgKWcffbZOTk52egxX375ZV7/+tc3esy2s08Wsj8Wsj9ONeo+efjhh3+amef02tfq0J+cnOShhx5q9JidTofp6elGj9l29slC9sdC9sepRt0nEfFfi+1zekeSKmLoS1JFDH1JqoihL0kVMfQlqSKGviRVxNCXpIoY+pJUEUNfkipi6EtSRQx9aRkm99416iZIq2LoS1JFDH1JqoihL0kVMfQlqSKGviRVxNCXpIoY+tIAXKqp9cLQl5bp8P+8NOomSCtm6EsD6h7tO/LXuDL0Jakihr4kVcTQl1Zocu9dTvNo7Bj6Uh8Gu9YTQ1+SKjJw6EfEaRHxSER8t2yfGxH3R8TRiPhGRLymlL+2bB8r+ye7HuP6Uv5URFwx7JORJC1tOSP964AjXdufBW7JzC3Ai8CuUr4LeDEz3wbcUuoRERcAO4C3A9uAL0XEaatrviRpOQYK/YjYBFwJ/GPZDuBS4JulygHgqnJ7e9mm7L+s1N8O3J6Zr2Tmj4FjwEXDOAlJ0mA2DFjvb4G/BN5Ytt8M/Cwz58r2DLCx3N4IPAuQmXMR8VKpvxG4r+sxu+/zGxGxG9gNMDExQafTGfRchmJ2drbxY7Zd7X2yZ+vcgu2J0xeW1dw34POjlzb3Sd/Qj4j3Ay9k5sMRMX2iuEfV7LNvqfv8tiBzH7APYGpqKqenp0+usqY6nQ5NH7Ptau6T+ZU7C/9M9myd4+bDXWWHX+aZm65stmEtUvPzYzFt7pNBRvrvAj4QEe8DXgf8DvMj/zMjYkMZ7W8Cniv1Z4DNwExEbADOAI53lZ/QfR9JUgP6zuln5vWZuSkzJ5l/I/b7mflh4F7gg6XaTuBguX1n2abs/35mZinfUVb3nAtsAR4Y2plIkvoadE6/l08At0fEZ4BHgFtL+a3A1yLiGPMj/B0AmflERNwBPAnMAddm5q9XcXxJ0jItK/QzswN0yu2n6bH6JjN/BVy9yP1vBG5cbiMlScPhJ3IlqSKGvjQEfj+PxoWhL0kVMfSlRTh613pk6EtSRQx9SaqIoS/14NSO1itDX5IqYuhLQ+KrA40DQ1+SKmLoS1JFDH1JqoihL0kVMfQlqSKGvnQSV+FoPTP0Jakihr4kVcTQl6SKGPqSVBFDX5IqYuhLQ+TKH7WdoS9JFTH0Jakihr7UxekZrXeGvjRkXjjUZoa+JFXE0Jekihj6klQRQ1+SKmLoS1JFDH1JqoihL0kVMfSlwvX1qoGhL60BLyBqK0Nfkipi6EtSRQx9SaqIoS9JFekb+hHxuoh4ICL+LSKeiIi/LuXnRsT9EXE0Ir4REa8p5a8t28fK/smux7q+lD8VEVes1UlJknobZKT/CnBpZv4h8A5gW0RcAnwWuCUztwAvArtK/V3Ai5n5NuCWUo+IuADYAbwd2AZ8KSJOG+bJSJKW1jf0c95s2Xx1+UngUuCbpfwAcFW5vb1sU/ZfFhFRym/PzFcy88fAMeCioZyFJGkgGwapVEbkDwNvA74I/Cfws8ycK1VmgI3l9kbgWYDMnIuIl4A3l/L7uh62+z7dx9oN7AaYmJig0+ks74xWaXZ2tvFjtl0tfbJn61z/SsDE6YPV/cJtB9m68YzVNqv1anl+LEeb+2Sg0M/MXwPviIgzge8A5/eqVn7HIvsWKz/5WPuAfQBTU1M5PT09SBOHptPp0PQx266GPpn/MNVAfw7s2TrHzYcHq/vMh6dX3qgxUcPzY7na3CfLWr2TmT8DOsAlwJkRceKZvwl4rtyeATYDlP1nAMe7y3vcR5LUgEFW75xTRvhExOnAu4EjwL3AB0u1ncDBcvvOsk3Z//3MzFK+o6zuORfYAjwwrBORJPU3yGvUtwIHyrz+q4A7MvO7EfEkcHtEfAZ4BLi11L8V+FpEHGN+hL8DIDOfiIg7gCeBOeDaMm0kSWpI39DPzMeAC3uUP02P1TeZ+Svg6kUe60bgxuU3U5I0DH4iV5IqYuhLUkUMfUmqiKGv6vkfnqgmhr4kVcTQl6SKGPqSVBFDX5IqYuhLUkUMfUmqiKEvSRUx9KU15GcA1DaGviRVxNCXpIoY+pJUEUNfVXPOXbUx9CWpIoa+JFXE0Jekihj6klQRQ19aY75ZrDYx9CWpIoa+JFXE0Jca4BSP2sLQl6SKGPqqlqNv1cjQl6SKGPqSVBFDX5IqYuhLUkUMfUmqiKGvKrlyR7Uy9KWGeKFRGxj6klQRQ1+SKmLoS1JFDH1JqoihL0kV6Rv6EbE5Iu6NiCMR8UREXFfK3xQRhyLiaPl9VimPiPh8RByLiMci4p1dj7Wz1D8aETvX7rSkxbmKRjUbZKQ/B+zJzPOBS4BrI+ICYC9wT2ZuAe4p2wDvBbaUn93Al2H+IgHcAFwMXATccOJCIUlqRt/Qz8znM/NH5fYvgCPARmA7cKBUOwBcVW5vB76a8+4DzoyItwJXAIcy83hmvggcArYN9WwkSUvasJzKETEJXAjcD0xk5vMwf2GIiLeUahuBZ7vuNlPKFis/+Ri7mX+FwMTEBJ1OZzlNXLXZ2dnGj9l2661P9mydW9X9J05f+WOsp348Yb09P4ahzX0ycOhHxBuAbwEfz8yfR8SiVXuU5RLlCwsy9wH7AKampnJ6enrQJg5Fp9Oh6WO23Xrrk2tWOae/Z+scNx9e1njptw6/zDM3Xbmq47fNent+DEOb+2Sg1TsR8WrmA/+2zPx2Kf5Jmbah/H6hlM8Am7vuvgl4bolySVJDBlm9E8CtwJHM/FzXrjuBEytwdgIHu8o/WlbxXAK8VKaB7gYuj4izyhu4l5cySVJDBnmN+i7gI8DhiHi0lP0VcBNwR0TsAv4buLrs+x7wPuAY8EvgTwEy83hEfBp4sNT7VGYeH8pZSANyuaZq1zf0M/Nf6T0fD3BZj/oJXLvIY+0H9i+ngZKk4fETuZJUEUNfkipi6EsN830FjZKhL0kVMfQlqSKGviRVxNBXNZxLlwx9SaqKoS9JFTH0pRFwqkmjYuirCoasNM/Ql6SKGPqSVBFDX5IqYuhLUkUMfUmqiKEvSRUx9CWpIoa+1j3X6Eu/ZehLUkUMfWlEfAWiUTD0pREy+NU0Q1+SKmLoa11zJC0tZOhLUkUMfUmqiKEvSRUx9KUR830HNcnQl6SKGPqSVBFDX5IqYuhLUkUMfUmqiKEvSRUx9LVujdNSyHFqq8aboa91yRCVejP0Jakihr4kVaRv6EfE/oh4ISIe7yp7U0Qcioij5fdZpTwi4vMRcSwiHouId3bdZ2epfzQidq7N6UhO7UhLGWSk/xVg20lle4F7MnMLcE/ZBngvsKX87Aa+DPMXCeAG4GLgIuCGExcKSVJz+oZ+Zv4AOH5S8XbgQLl9ALiqq/yrOe8+4MyIeCtwBXAoM49n5ovAIU69kEiS1tiGFd5vIjOfB8jM5yPiLaV8I/BsV72ZUrZY+SkiYjfzrxKYmJig0+mssIkrMzs72/gx227c+mTP1rk1ffyJ09fmGF+47SBbN54x9Mdda+P2/GhCm/tkpaG/mOhRlkuUn1qYuQ/YBzA1NZXT09NDa9wgOp0OTR+z7catT65Z4zn9PVvnuPnwsP905j3z4ek1edy1NG7Pjya0uU9WunrnJ2XahvL7hVI+A2zuqrcJeG6JcklSg1Ya+ncCJ1bg7AQOdpV/tKziuQR4qUwD3Q1cHhFnlTdwLy9l0lC5ckda2iBLNr8O/BA4LyJmImIXcBPwnog4CrynbAN8D3gaOAb8A/AXAJl5HPg08GD5+VQpk9TFi5bWWt+Jycz80CK7LutRN4FrF3mc/cD+ZbVOkjRUfiJXkipi6Est4xSP1pKhL0kVMfS1bjhClvoz9CWpIoa+JFXE0Ne64NSONBhDX2ohL2JaK4a+JFXE0JdaytG+1oKhr7FnOEqDM/QlqSKGviRVxNCXWsypKw2boa+xZSBKy2foa6zVEPw1nKOaY+hLUkUMfY0lR7/Syhj6klQRQ18aA76y0bAY+ho7BqC0coa+NCa82GkYDH1JqoihL40RR/taLUNfY8XQk1bH0NfYMPCl1dsw6gZI/Rj2C53oj2duunLELdE4cqQvSRUx9NVqjvIXZ99oJQx9tZah1p99pOUy9NVKhtng7Csth6EvSRVx9Y5axVGrtLYc6UvrwOTeu7xgaiCGvlrD0Fo9+1D9OL2jkTKkhm9y711+cEuLcqSvkTDs19aJ/rWfdTJDX43qnns2kNaW/axeGg/9iNgWEU9FxLGI2Nv08dWMk99Y9I3G0er176E6NTqnHxGnAV8E3gPMAA9GxJ2Z+WST7dDwdH/5V68gMVzapVfwO/9fl6bfyL0IOJaZTwNExO3AdsDQX2Mnv7l38h/8UuG8Z+sc1/QJb8N9fA36b9d9YfdCMb4iM5s7WMQHgW2Z+edl+yPAxZn5sa46u4HdZfM84KnGGjjvbOCnDR+z7eyTheyPheyPU426T34vM8/ptaPpkX70KFtw1cnMfcC+Zppzqoh4KDOnRnX8NrJPFrI/FrI/TtXmPmn6jdwZYHPX9ibguYbbIEnVajr0HwS2RMS5EfEaYAdwZ8NtkKRqNTq9k5lzEfEx4G7gNGB/Zj7RZBsGMLKppRazTxayPxayP07V2j5p9I1cSdJo+YlcSaqIoS9JFTH0e4iIv4mIf4+IxyLiOxFx5qjbNEoRcXVEPBER/xcRrVyG1gS/QmShiNgfES9ExOOjbksbRMTmiLg3Io6Uv5frRt2mXgz93g4Bv5+ZfwD8B3D9iNszao8DfwL8YNQNGZWurxB5L3AB8KGIuGC0rRq5rwDbRt2IFpkD9mTm+cAlwLVtfI4Y+j1k5j9n5lzZvI/5zxNUKzOPZGbTn4xum998hUhm/i9w4itEqpWZPwCOj7odbZGZz2fmj8rtXwBHgI2jbdWpDP3+/gz4p1E3QiO3EXi2a3uGFv5Bqx0iYhK4ELh/tC05VbX/c1ZE/Avwuz12fTIzD5Y6n2T+JdttTbZtFAbpj8r1/QoRCSAi3gB8C/h4Zv581O05WbWhn5nvXmp/ROwE3g9clhV8mKFff8ivEFF/EfFq5gP/tsz89qjb04vTOz1ExDbgE8AHMvOXo26PWsGvENGSIiKAW4Ejmfm5UbdnMYZ+b38HvBE4FBGPRsTfj7pBoxQRfxwRM8AfAXdFxN2jblPTyhv7J75C5AhwRwu/QqRREfF14IfAeRExExG7Rt2mEXsX8BHg0pIbj0bE+0bdqJP5NQySVBFH+pJUEUNfkipi6EtSRQx9SaqIoS9JFTH0Jakihr4kVeT/AfYHhOdhgDdxAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ax = pd.Series(weightvalues).hist(bins=1000)\n",
    "#ax.set_xlim(-0.2, 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD4CAYAAAAO9oqkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAULklEQVR4nO3df4zc9Z3f8efr7JDkoDmTkKyobdVUZ11DkgaSFbhFqjYhBUNOZ05KJFAbnITK1xO0SYXUQKqKNASVqJekR5ug+oIb06NxEEmExTnhXI7VKdJBMIECxkfZAoUFCpfaEEzaIKfv/jEfX+eW2R/eHe/sDM+HNJqZ9/fz/eznLdvzmu93vrNOVSFJemP7lUEvQJI0eIaBJMkwkCQZBpIkDANJErB60AtYrFNOOaU2bNjQt/leffVVTjzxxL7Nt1KMYl/2NDxGsa9h7+n+++//aVW9c2Z9aMNgw4YN7Nu3r2/zTU5OMjEx0bf5VopR7Muehsco9jXsPSX5H73qniaSJBkGkiTDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgSWIBYZDkLUl+nOS/Jtmf5F+1+mlJ7k3yeJLvJDmh1d/cnk+17Ru65rq61R9Lcn5XfXOrTSW5qv9tSpLmspAjg18AH66q9wNnAJuTbAK+DHytqjYCh4DL2vjLgENV9evA19o4kpwOXAy8B9gMfCPJqiSrgK8DFwCnA5e0sZKkZTJvGFTH4fb0Te1WwIeB21p9J3BRe7ylPadtPzdJWn1XVf2iqp4EpoCz2m2qqp6oqteAXW2sJGmZLOj/M2jv3u8Hfp3Ou/j/DrxUVUfakGlgbXu8FngGoKqOJHkZeEer39M1bfc+z8yonz3LOrYB2wDGxsaYnJxcyPIX5PDhw32db6UYxb7saXiMYl+j2BMsMAyq6pfAGUnWAN8H3t1rWLvPLNtmq/c6OqkeNapqO7AdYHx8vPr5H0wM+39YMZtR7Muehsco9jWKPcExXk1UVS8Bk8AmYE2So2GyDniuPZ4G1gO07b8GHOyuz9hntrokaZks5Gqid7YjApK8FfgIcAC4G/hYG7YVuL093t2e07b/SVVVq1/crjY6DdgI/Bi4D9jYrk46gc6HzLv70ZwkaWEWcproVGBn+9zgV4Bbq+qOJI8Cu5J8CXgAuKmNvwn4T0mm6BwRXAxQVfuT3Ao8ChwBLm+nn0hyBXAnsArYUVX7+9ahJGle84ZBVT0EnNmj/gSdK4Fm1v8P8PFZ5roOuK5HfQ+wZwHrlSQdB34DWZJkGEiSDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEksIAySrE9yd5IDSfYn+UyrfyHJs0kebLcLu/a5OslUkseSnN9V39xqU0mu6qqfluTeJI8n+U6SE/rdqCRpdgs5MjgCXFlV7wY2AZcnOb1t+1pVndFuewDatouB9wCbgW8kWZVkFfB14ALgdOCSrnm+3ObaCBwCLutTf5KkBZg3DKrq+ar6SXv8CnAAWDvHLluAXVX1i6p6EpgCzmq3qap6oqpeA3YBW5IE+DBwW9t/J3DRYhuSJB271ccyOMkG4EzgXuAc4IoklwL76Bw9HKITFPd07TbN/w+PZ2bUzwbeAbxUVUd6jJ/587cB2wDGxsaYnJw8luXP6fDhw32db6UYxb7saXiMYl+j2BMcQxgkOQn4LvDZqvpZkhuBa4Fq918BPg2kx+5F76OQmmP864tV24HtAOPj4zUxMbHQ5c9rcnKSfs63UoxiX/Y0PEaxr1HsCRYYBkneRCcIbqmq7wFU1Qtd2/8AuKM9nQbWd+2+DniuPe5V/ymwJsnqdnTQPV6StAwWcjVRgJuAA1X11a76qV3Dfht4pD3eDVyc5M1JTgM2Aj8G7gM2tiuHTqDzIfPuqirgbuBjbf+twO1La0uSdCwWcmRwDvAJ4OEkD7ba5+lcDXQGnVM6TwG/A1BV+5PcCjxK50qky6vqlwBJrgDuBFYBO6pqf5vvc8CuJF8CHqATPpKkZTJvGFTVj+h9Xn/PHPtcB1zXo76n135V9QSdq40kSQPgN5AlSYaBJMkwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSSJBYRBkvVJ7k5yIMn+JJ9p9bcn2Zvk8XZ/cqsnyQ1JppI8lOQDXXNtbeMfT7K1q/7BJA+3fW5IkuPRrCSpt4UcGRwBrqyqdwObgMuTnA5cBdxVVRuBu9pzgAuAje22DbgROuEBXAOcDZwFXHM0QNqYbV37bV56a5KkhZo3DKrq+ar6SXv8CnAAWAtsAXa2YTuBi9rjLcDN1XEPsCbJqcD5wN6qOlhVh4C9wOa27W1V9WdVVcDNXXNJkpbB6mMZnGQDcCZwLzBWVc9DJzCSvKsNWws807XbdKvNVZ/uUe/187fROYJgbGyMycnJY1n+nA4fPtzX+VaKUezLnobHKPY1ij3BMYRBkpOA7wKfraqfzXFav9eGWkT99cWq7cB2gPHx8ZqYmJhn1Qs3OTlJP+dbKUaxL3saHqPY1yj2BAu8mijJm+gEwS1V9b1WfqGd4qHdv9jq08D6rt3XAc/NU1/Xoy5JWiYLuZoowE3Agar6atem3cDRK4K2Ard31S9tVxVtAl5up5PuBM5LcnL74Pg84M627ZUkm9rPurRrLknSMljIaaJzgE8ADyd5sNU+D1wP3JrkMuBp4ONt2x7gQmAK+DnwKYCqOpjkWuC+Nu6LVXWwPf5d4FvAW4EftJskaZnMGwZV9SN6n9cHOLfH+AIun2WuHcCOHvV9wHvnW4sk6fjwG8iSJMNAkmQYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJLEAsIgyY4kLyZ5pKv2hSTPJnmw3S7s2nZ1kqkkjyU5v6u+udWmklzVVT8tyb1JHk/ynSQn9LNBSdL8FnJk8C1gc4/616rqjHbbA5DkdOBi4D1tn28kWZVkFfB14ALgdOCSNhbgy22ujcAh4LKlNCRJOnbzhkFV/SlwcIHzbQF2VdUvqupJYAo4q92mquqJqnoN2AVsSRLgw8Btbf+dwEXH2IMkaYmW8pnBFUkeaqeRTm61tcAzXWOmW222+juAl6rqyIy6JGkZrV7kfjcC1wLV7r8CfBpIj7FF79CpOcb3lGQbsA1gbGyMycnJY1r0XA4fPtzX+VaKUezLnobHKPY1ij3BIsOgql44+jjJHwB3tKfTwPquoeuA59rjXvWfAmuSrG5HB93je/3c7cB2gPHx8ZqYmFjM8nuanJykn/OtFKPYlz0Nj1HsaxR7gkWeJkpyatfT3waOXmm0G7g4yZuTnAZsBH4M3AdsbFcOnUDnQ+bdVVXA3cDH2v5bgdsXsyZJ0uLNe2SQ5NvABHBKkmngGmAiyRl0Tuk8BfwOQFXtT3Ir8ChwBLi8qn7Z5rkCuBNYBeyoqv3tR3wO2JXkS8ADwE19606StCDzhkFVXdKjPOsLdlVdB1zXo74H2NOj/gSdq40kSQPiN5AlSYaBJMkwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CSxALCIMmOJC8meaSr9vYke5M83u5PbvUkuSHJVJKHknyga5+tbfzjSbZ21T+Y5OG2zw1J0u8mJUlzW8iRwbeAzTNqVwF3VdVG4K72HOACYGO7bQNuhE54ANcAZwNnAdccDZA2ZlvXfjN/liTpOJs3DKrqT4GDM8pbgJ3t8U7goq76zdVxD7AmyanA+cDeqjpYVYeAvcDmtu1tVfVnVVXAzV1zSZKWyepF7jdWVc8DVNXzSd7V6muBZ7rGTbfaXPXpHvWekmyjcxTB2NgYk5OTi1z+6x0+fLiv860Uo9iXPQ2PUexrFHuCxYfBbHqd769F1Huqqu3AdoDx8fGamJhYxBJ7m5ycpJ/zrRSj2Jc9DY9R7GsUe4LFX030QjvFQ7t/sdWngfVd49YBz81TX9ejLklaRosNg93A0SuCtgK3d9UvbVcVbQJebqeT7gTOS3Jy++D4PODOtu2VJJvaVUSXds0lSVom854mSvJtYAI4Jck0nauCrgduTXIZ8DTw8TZ8D3AhMAX8HPgUQFUdTHItcF8b98WqOvqh9O/SuWLprcAP2k2StIzmDYOqumSWTef2GFvA5bPMswPY0aO+D3jvfOuQJB0/fgNZkmQYSJIMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCQBqwe9gDeyDVf90V95/tT1Hx3QSiS90S3pyCDJU0keTvJgkn2t9vYke5M83u5PbvUkuSHJVJKHknyga56tbfzjSbYurSVJ0rHqx2miD1XVGVU13p5fBdxVVRuBu9pzgAuAje22DbgROuEBXAOcDZwFXHM0QN5oZh4pSNJyOR6fGWwBdrbHO4GLuuo3V8c9wJokpwLnA3ur6mBVHQL2ApuPw7pWFF/4Ja0kqarF75w8CRwCCvgPVbU9yUtVtaZrzKGqOjnJHcD1VfWjVr8L+BwwAbylqr7U6v8S+N9V9Xs9ft42OkcVjI2NfXDXrl2LXvtMhw8f5qSTTurbfHN5+NmXZ932vrW/1teftZx9LRd7Gh6j2New9/ShD33o/q4zOX9pqR8gn1NVzyV5F7A3yZ/PMTY9ajVH/fXFqu3AdoDx8fGamJg4xuXObnJykn7ON5dPznFU8NQ/6O8alrOv5WJPw2MU+xrFnmCJp4mq6rl2/yLwfTrn/F9op39o9y+24dPA+q7d1wHPzVGXJC2TRYdBkhOT/LWjj4HzgEeA3cDRK4K2Are3x7uBS9tVRZuAl6vqeeBO4LwkJ7cPjs9rtTckP0uQNAhLOU00Bnw/ydF5/nNV/TDJfcCtSS4DngY+3sbvAS4EpoCfA58CqKqDSa4F7mvjvlhVB5ewLknSMVp0GFTVE8D7e9T/F3Buj3oBl88y1w5gx2LXMkx85y9pJfLXUUiSDIOVyKMHScvNMFhGvshLWqkMA0mSYSBJMgyWzbGeIvKUkqTlZBhIkgyD5eC7fEkrnWFwHBkCkoaFYXCcHA2CpQSCYSJpuRgGkiTDQJJkGBwXnt6RNGwMgz7rdxAYLJKWg2EgSTIMJEmGwVDwVJGk480w6CNftCUNK8NgSBg0ko4nw0CSZBj0y3K8c/foQNLxYhgMmQ1X/ZGhIKnvDIM+GMSLs6EgqZ8MgyFnKEjqh9WDXsCwWykvxLOt48r3HWGia8xT13902dYkaXismDBIshn4fWAV8M2qun7AS5rXSgmC+XSvc641dweFwSGtPMfz3+WKCIMkq4CvA38fmAbuS7K7qh4d7MpmNyxBcCxm9nQ8enzq+o/+5V/oo/PPDKHucTO3Azz87Mt8cp45jj6f+Y/HkNOwOt6vOamq4/oDFrSI5O8AX6iq89vzqwGq6l/Pts/4+Hjt27evb2v4d7fczlceXhHZ2FdXvu/IyPVlT8Njrr66w3yhFrPPYvR6g3LUtzafyCd/+Oqi513q+pf6ZibJ/VU1/rr6CgmDjwGbq+ofteefAM6uqitmjNsGbGtPfwN4rI/LOAX4aR/nWylGsS97Gh6j2New9/Q3quqdM4sr5a1IetRel1JVtR3YflwWkOzrlZbDbhT7sqfhMYp9jWJPsHIuLZ0G1nc9Xwc8N6C1SNIbzkoJg/uAjUlOS3ICcDGwe8BrkqQ3jBVxmqiqjiS5AriTzqWlO6pq/zIv47icfloBRrEvexoeo9jXKPa0Mj5AliQN1ko5TSRJGiDDQJJkGHRL8m+S/HmSh5J8P8maQa9psZJsTvJYkqkkVw16PUuVZH2Su5McSLI/yWcGvaZ+SrIqyQNJ7hj0WvohyZokt7V/TwfaF0uHWpJ/1v7uPZLk20neMug19ZNh8FftBd5bVX8b+G/A1QNez6J0/XqPC4DTgUuSnD7YVS3ZEeDKqno3sAm4fAR66vYZ4MCgF9FHvw/8sKr+FvB+hry3JGuBfwqMV9V76VzocvFgV9VfhkGXqvrjqjrSnt5D5/sOw+gsYKqqnqiq14BdwJYBr2lJqur5qvpJe/wKnReXtYNdVX8kWQd8FPjmoNfSD0neBvw94CaAqnqtql4a7Kr6YjXw1iSrgV9lxL4LZRjM7tPADwa9iEVaCzzT9XyaEXnhBEiyATgTuHewK+mbfwv8c+D/DnohffI3gb8A/mM79fXNJCcOelFLUVXPAr8HPA08D7xcVX882FX11xsuDJL8l3bOb+ZtS9eYf0HntMQtg1vpkizo13sMoyQnAd8FPltVPxv0epYqyW8CL1bV/YNeSx+tBj4A3FhVZwKvAkP9uVWSk+kcXZ8G/HXgxCT/cLCr6q8V8aWz5VRVH5lre5KtwG8C59bwfgljJH+9R5I30QmCW6rqe4NeT5+cA/xWkguBtwBvS/KHVTXMLzTTwHRVHT1yu40hDwPgI8CTVfUXAEm+B/xd4A8Huqo+esMdGcyl/Qc7nwN+q6p+Puj1LMHI/XqPJKFzDvpAVX110Ovpl6q6uqrWVdUGOn9OfzLkQUBV/U/gmSS/0UrnAiv2/yZZoKeBTUl+tf1dPJch/1B8pjfckcE8/j3wZmBv58+be6rqHw92Scduhfx6j347B/gE8HCSB1vt81W1Z4Br0uz+CXBLezPyBPCpAa9nSarq3iS3AT+hcwr5AUbs11L46ygkSZ4mkiQZBpIkDANJEoaBJAnDQJKEYSBJwjCQJAH/Dwp4DuE9MEcOAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ax = pd.Series(np.log(sqrtsigmavalues)).hist(bins=1000)\n",
    "#ax.set_xlim((0.00, 120))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD4CAYAAAAEhuazAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAZbklEQVR4nO3dcZCc9X3f8ffHyICHcyQRzEWR1J4YK6kBjW10A6Q0nTtkC4HTiLaQKsME4dDRpMEZZ4ZMEeM6uIAncmPHtWdiXMVSIztOzlQJQYNwiCpz9bgzwlgGI4SgOrDGPkmVakvIOUPIKPn2j/2dszp2b5+7233u2f19XjM7++xvf88+3+fZvc/z7G+f3VNEYGZmeXnLfBdgZmblc/ibmWXI4W9mliGHv5lZhhz+ZmYZWjDfBUznkksuiYGBgVKX+eMf/5iLLrqo1GW2UsWaoJp1VbEmcF0zUcWaoJp1Natp//79P4iId0w7c0RU9rJ69eoo25NPPln6MlupYk0R1ayrijVFuK6ZqGJNEdWsq1lNwLeiRb562MfMLEMOfzOzDDn8zcwy5PA3M8uQw9/MLEMOfzOzDDn8zcwy5PA3M8uQw9/MLEMOfzPLxsDm3Qxs3j3fZVSCw9/MsuDQP5fD38yylPvOoNK/6mlmNleNQj734Acf+ZuZZcnhb2bZyvkdgMPfzCxDDn8zswwVCn9JiyTtlPSipEOSfkHSxZL2SDqcrhenvpL0WUljkp6TdFXd42xM/Q9L2tiplTIzs+kVPfL/DPBXEfHPgHcDh4DNwN6IWAnsTbcBbgRWpssm4CEASRcD9wHXAFcD903uMMzMrFwtw1/STwH/EtgGEBF/FxGvAuuBHanbDuDmNL0e+GL6V5L7gEWSlgA3AHsi4lREnAb2AOvaujZmZnVy/kC3lSJH/pcB/w/475KekfQFSRcB/RFxHCBdX5r6LwW+Xzf/eGpr1m5mZiVT7R+9T9NBGgT2AddFxFOSPgP8CPitiFhU1+90RCyWtBv4vYj4RmrfC/xH4Hrggoh4MLV/FHgtIj41ZXmbqA0X0d/fv3pkZKRNq1rMxMQEfX19pS6zlSrWBNWsq4o1geuaiXbWdODomZZ9Vi1dWOixumlbDQ8P74+IwenmLfIN33FgPCKeSrd3UhvfPyFpSUQcT8M6J+v6L6+bfxlwLLUPTWkfnbqwiNgKbAUYHByMoaGhqV06anR0lLKX2UoVa4Jq1lXFmsB1zUQ7a7qjwLDPkduKLavXtlXLYZ+I+L/A9yX9fGpaA7wA7AImz9jZCDyapncBt6ezfq4FzqRhoSeAtZIWpw9616Y2MzMrWdHf9vkt4MuSzgdeAT5IbcfxsKQ7ge8Bt6a+jwM3AWPAa6kvEXFK0gPA06nf/RFxqi1rYWY2SwObd3Nkywfmu4zSFQr/iHgWaDR+tKZB3wDuavI424HtMynQzMzaz9/wNTPLkMPfzHqSz/GfnsPfzLKX447C4W9mliGHv5lZhhz+ZmYZcvibmWXI4W9mliGHv5lZhhz+Zmbkd7qnw9/MLEMOfzPrObkdxc+Gw9/MLEMOfzOzDDn8zcwy5PA3M8uQw9/MLENF/42jmVnl+Syf4nzkb2aWIYe/mVmS0zsHh7+ZWYYc/mZmGXL4m5llyOFvZpahQuEv6YikA5KelfSt1HaxpD2SDqfrxaldkj4raUzSc5Kuqnucjan/YUkbO7NKZmbWykyO/Icj4j0RMZhubwb2RsRKYG+6DXAjsDJdNgEPQW1nAdwHXANcDdw3ucMwM5urnM7UaYe5DPusB3ak6R3AzXXtX4yafcAiSUuAG4A9EXEqIk4De4B1c1i+mZnNkiKidSfpu8BpIID/FhFbJb0aEYvq+pyOiMWSHgO2RMQ3Uvte4B5gCLgwIh5M7R8FXo+IT05Z1iZq7xjo7+9fPTIy0obVLG5iYoK+vr5Sl9lKFWuCatZVxZrAdc3EbGs6cPRMW5a/aunChu3dtK2Gh4f3143SNFT05x2ui4hjki4F9kh6cZq+atAW07Sf2xCxFdgKMDg4GENDQwVLbI/R0VHKXmYrVawJqllXFWsC1zUTs63pjjYN+xy5rfGye2lbQcFhn4g4lq5PAo9QG7M/kYZzSNcnU/dxYHnd7MuAY9O0m5lZyVqGv6SLJL19chpYCzwP7AImz9jZCDyapncBt6ezfq4FzkTEceAJYK2kxemD3rWpzczMSlZk2KcfeETSZP8/jYi/kvQ08LCkO4HvAbem/o8DNwFjwGvABwEi4pSkB4CnU7/7I+JU29bEzMwKaxn+EfEK8O4G7T8E1jRoD+CuJo+1Hdg+8zLNzKyd/A1fM7MMOfzNzDLk8Dczy5DD38wsQw5/M+t6/l2fmXP4m5llyOFvZl2t3Uf9ubyLcPibmWXI4W9mliGHv5lZhhz+ZmYZcvibWdfK5cPZTnD4m5llyOFvZjZFDu8oHP5mZhly+JuZZcjhb2aWIYe/mXWlHMblO8nhb2aWIYe/mVmGHP5mZhly+JtZ1/F4/9w5/M3MMuTwNzPLUOHwl3SepGckPZZur5D0lKTDkr4i6fzUfkG6PZbuH6h7jHtT+0uSbmj3ypiZWTEzOfL/MHCo7vYngE9HxErgNHBnar8TOB0R7wQ+nfoh6XJgA3AFsA74nKTz5la+mZnNRqHwl7QM+ADwhXRbwPXAztRlB3Bzml6fbpPuX5P6rwdGIuKNiPguMAZc3Y6VMDOzmVFEtO4k7QR+D3g78DvAHcC+dHSPpOXAVyPiSknPA+siYjzd9zJwDfCxNM+fpPZtaZ6dU5a1CdgE0N/fv3pkZKQNq1ncxMQEfX19pS6zlSrWBNWsq4o1geuaiSI1HTh6puN1rFq68Jzb3bSthoeH90fE4HTzLmj14JJ+CTgZEfslDU02N+gaLe6bbp5/bIjYCmwFGBwcjKGhoaldOmp0dJSyl9lKFWuCatZVxZrAdc1EkZruKOFUzyO3nVtDt26rZlqGP3Ad8MuSbgIuBH4K+K/AIkkLIuIssAw4lvqPA8uBcUkLgIXAqbr2SfXzmJlZiVqO+UfEvRGxLCIGqH1g+7WIuA14ErglddsIPJqmd6XbpPu/FrWxpV3AhnQ20ApgJfDNtq2JmWXBX/BqjyJH/s3cA4xIehB4BtiW2rcBX5I0Ru2IfwNARByU9DDwAnAWuCsi/n4OyzezzDj422dG4R8Ro8Bomn6FBmfrRMTfArc2mf/jwMdnWqSZmbWXv+FrZtZAr7/LcPibmWXI4W9mliGHv5lZhhz+ZmYZcvibWVfo9Q9gy+bwNzPLkMPfzCxDDn8zsww5/M3MMuTwNzPLkMPfzCxDDn8zsww5/M3MMuTwNzPLkMPfzCrP3+5tP4e/mVkTvbzTcfibWaX1cgDPJ4e/mVmGHP5mZhly+JuZZcjhb2aWIYe/mVVSVT7orUod7dYy/CVdKOmbkr4j6aCk/5zaV0h6StJhSV+RdH5qvyDdHkv3D9Q91r2p/SVJN3RqpczMbHpFjvzfAK6PiHcD7wHWSboW+ATw6YhYCZwG7kz97wROR8Q7gU+nfki6HNgAXAGsAz4n6bx2royZmRXTMvyjZiLdfGu6BHA9sDO17wBuTtPr023S/WskKbWPRMQbEfFdYAy4ui1rYWY9qVeHXKpAEdG6U+0IfT/wTuAPgd8H9qWjeyQtB74aEVdKeh5YFxHj6b6XgWuAj6V5/iS1b0vz7JyyrE3AJoD+/v7VIyMj7VjPwiYmJujr6yt1ma1UsSaoZl1VrAlc10xM1nTg6Jn5LuUnVi1dWOltNdXw8PD+iBicbt4FRRYQEX8PvEfSIuAR4F2NuqVrNbmvWfvUZW0FtgIMDg7G0NBQkRLbZnR0lLKX2UoVa4Jq1lXFmsB1zcRkTXdU6Kj/yG1Dld5WszGjs30i4lVgFLgWWCRpcuexDDiWpseB5QDp/oXAqfr2BvOYmVmJipzt8450xI+ktwHvAw4BTwK3pG4bgUfT9K50m3T/16I2trQL2JDOBloBrAS+2a4VMTOz4ooM+ywBdqRx/7cAD0fEY5JeAEYkPQg8A2xL/bcBX5I0Ru2IfwNARByU9DDwAnAWuCsNJ5mZWclahn9EPAe8t0H7KzQ4Wyci/ha4tcljfRz4+MzLNDOzdvI3fM2scnyKZ+c5/M3MMuTwN7NKqdL5/b3M4W9mliGHv5lZhhz+ZmYZcvibmWXI4W9mliGHv5lVRlXP769qXXPh8Dczy5DD38wsQw5/M6uEXhxaqTKHv5lZhhz+ZmYZcvibmWXI4W9m887j/eVz+JuZFdBrvzbq8Dczy5DD38wsQw5/M7MMOfzNbF75w9754fA3s3nj4J8/Dn8zsww5/M1sXviof361DH9JyyU9KemQpIOSPpzaL5a0R9LhdL04tUvSZyWNSXpO0lV1j7Ux9T8saWPnVsvMzKZT5Mj/LHB3RLwLuBa4S9LlwGZgb0SsBPam2wA3AivTZRPwENR2FsB9wDXA1cB9kzsMMzMrV8vwj4jjEfHtNP03wCFgKbAe2JG67QBuTtPrgS9GzT5gkaQlwA3Anog4FRGngT3AuraujZmZFaKIKN5ZGgC+DlwJfC8iFtXddzoiFkt6DNgSEd9I7XuBe4Ah4MKIeDC1fxR4PSI+OWUZm6i9Y6C/v3/1yMjIrFduNiYmJujr6yt1ma1UsSaoZl1VrAlcVyPNfi6h/21w4vWSiymg/21w6cUL57uMczR7/oaHh/dHxOB08y4ouhBJfcCfA78dET+S1LRrg7aYpv3choitwFaAwcHBGBoaKlpiW4yOjlL2MlupYk1QzbqqWBO4rkbuaPKB792rzvKpA4WjqTR3rzrLr1TsOZzL81fobB9Jb6UW/F+OiL9IzSfScA7p+mRqHweW182+DDg2TbuZZcZn+sy/Imf7CNgGHIqIP6i7axcwecbORuDRuvbb01k/1wJnIuI48ASwVtLi9EHv2tRmZmYlK3Lkfx3wa8D1kp5Nl5uALcD7JR0G3p9uAzwOvAKMAX8E/CZARJwCHgCeTpf7U5uZZaSbj/q7ufapWg6spQ9umw3wr2nQP4C7mjzWdmD7TAo0M7P28zd8zaw0vXTk3O0c/mZWCgd/tTj8zcwy5PA3M8uQw9/MLEMOfzOzDFXvO9Rm1lP8QW81+cjfzCxDDn8z6xgf9VeXw9/MLEMOfzNru14+4u+VdXP4m1lH9EpI9iqHv5lZhhz+ZmYZcvibWdvkMtTTC+vp8DeztuqFYMyBw9/MLEMOfzOzDDn8zWzOPNTTfRz+ZtYW3gF0F/+qp5nNmgO/e/nI38wsQw5/M7MMtQx/SdslnZT0fF3bxZL2SDqcrhendkn6rKQxSc9Juqpuno2p/2FJGzuzOmZWBg/3dP82KHLk/8fAuiltm4G9EbES2JtuA9wIrEyXTcBDUNtZAPcB1wBXA/dN7jDMrLtMhl63h1/uWoZ/RHwdODWleT2wI03vAG6ua/9i1OwDFklaAtwA7ImIUxFxGtjDm3coZmZWEkVE607SAPBYRFyZbr8aEYvq7j8dEYslPQZsiYhvpPa9wD3AEHBhRDyY2j8KvB4Rn2ywrE3U3jXQ39+/emRkZE4rOFMTExP09fWVusxWqlgTVLOuKtYEvVXXgaNnOlRNTf/b4MTrHV3ErDSqa9XShfNTTNLs+RseHt4fEYPTzdvuUz3VoC2maX9zY8RWYCvA4OBgDA0Nta24IkZHRyl7ma1UsSaoZl1VrAl6p67aUE9nzxC/e9VZPnWgemehN6rryG1D81NMMpfX1Wy38AlJSyLieBrWOZnax4Hldf2WAcdS+9CU9tFZLtvMSubx/d4z21M9dwGTZ+xsBB6ta789nfVzLXAmIo4DTwBrJS1OH/SuTW1mVnEO/t7U8shf0p9RO2q/RNI4tbN2tgAPS7oT+B5wa+r+OHATMAa8BnwQICJOSXoAeDr1uz8ipn6IbGYV4tDvbS3DPyJ+tcldaxr0DeCuJo+zHdg+o+rMzCpsYPNujmz5wHyXMSv+hq+ZvYmP+nufw9/MzuHgz0P1zqeyntcoXLr1rXOvcfDPXLcO/Tj8rRSTodKNfyQ5cOjnx8M+VqpmIePfi5k/3uZ58pG/ddRMgsUhVL7aTzU4BnLkZ906xmFeXZPPzd2r5rkQmzce9rG2m2voe6fRWd6+7deN29Thb23TjX8AufFzZJMc/tZW7frgdmDzbgfVHE19Lrw9rZ7H/G3OHCrV5+fIpnL4m/WQqSHv0LdmPOxjs3bg6JmOh4vDa3r1QzreVjYTDn+blTKDxqHWmMfybS4c/jZjDpvy1W9zH+VXU7c9Lx7zN+sS3RQsVn0+8rdC5nuIIbfgqz+KzG3drRwOf2upSiFUhRraqdW5+L22vjnolufM4W9NVW0Ms0q1zEWzwK/STtZ6n8PfGnIAzUyjUy4npwc2737TabHevr2tG55ff+BrQHe8WCfN1z+GabWNioR7N21nm5uq/4cvh3/mujmMptZ+ZMsHOHD0DEOc+4c3OV2/05g6Xd82tY/ZbFV5B+Dwz0wvB9rA5t3cvarx2HmraX/Qap1S1R2Ax/x7nL/+b2aNlH7kL2kd8BngPOALEbGl7Bp6lceZzaqpfmixKu8CSg1/SecBfwi8HxgHnpa0KyJeKLOObnPg6BnucICbdbX6d+FV2AGUfeR/NTAWEa8ASBoB1gNdH/5Tn9BGHy7Olv/Pqllvma8z1uopIspbmHQLsC4i/n26/WvANRHxobo+m4BN6ebPAy+VVmDNJcAPSl5mK1WsCapZVxVrAtc1E1WsCapZV7Oa/mlEvGO6Gcs+8leDtnP2PhGxFdhaTjlvJulbETE4X8tvpIo1QTXrqmJN4Lpmooo1QTXrmktNZZ/tMw4sr7u9DDhWcg1mZtkrO/yfBlZKWiHpfGADsKvkGszMslfqsE9EnJX0IeAJaqd6bo+Ig2XWUMC8DTlNo4o1QTXrqmJN4Lpmooo1QTXrmnVNpX7ga2Zm1eBv+JqZZcjhb2aWoezCX9Ktkg5K+gdJg3Xtt0l6tu7yD5Le02D+j0k6Wtfvpg7XNSDp9brlfb7J/BdL2iPpcLpe3OG63i9pv6QD6fr6JvO3fXs1qyndd6+kMUkvSbqhyfwrJD2VttVX0skHbZUed3Kdj0h6tkm/I2kbPivpW+2uo8HyCj0fktalbTgmaXOHa/p9SS9Kek7SI5IWNelXyrZqte6SLkjP71h6HQ10qpa0vOWSnpR0KL3uP9ygz5CkM3XP6++2fOCIyOoCvIval8dGgcEmfVYBrzS572PA75RVFzAAPF9g/v8CbE7Tm4FPdLiu9wI/m6avBI6Wtb2mqely4DvABcAK4GXgvAbzPwxsSNOfB/5Dh19znwJ+t8l9R4BLOrn8mT4f1E7GeBm4DDg/bdPLO1jTWmBBmv5Es9duGduqyLoDvwl8Pk1vAL7S4ZqWAFel6bcD/6dBTUPAYzN53OyO/CPiUES0+tbwrwJ/VkY9kwrWNZ31wI40vQO4ee5VNa8rIp6JiMnvaBwELpR0QTuWOduaqG2DkYh4IyK+C4xR+0mRn5Ak4HpgZ2pq27ZqJC3vVyj59TRHP/kZloj4O2DyZ1g6IiL+OiLOppv7qH3/Z74UWff6v7WdwJr0PHdERByPiG+n6b8BDgFL5/q42YV/Qf+O6f9YP5Teom5v1/BKCyskPSPpf0n6xSZ9+iPiONReLMClJdQ16d8Cz0TEG03uL2t7LQW+X3d7nDf/kfw08Gpd2DTq006/CJyIiMNN7g/gr9PQ2aYmfdqt1fNRZDt2yq8DX21yXxnbqsi6/6RPeh2dofa66rg0xPRe4KkGd/+CpO9I+qqkK1o9Vk/+MxdJ/xP4mQZ3fSQiHm0x7zXAaxHxfJMuDwEPUHshPkDtLf2vd7Cu48A/iYgfSloN/KWkKyLiR0WW2cG6Jue9gtpb9bVNusxqe82yppY/H1KwTyEFa2z1LvK6iDgm6VJgj6QXI+Lrs6mnSF0Uez7ato2K1DS5rSR9BDgLfLnJw7R9WzUqtUFbx15DMyGpD/hz4Lcb/P1/m9rv+Uykz3H+Elg53eP1ZPhHxPvmMPsGpvljjYgTk9OS/gh4rJN1paPpN9L0fkkvAz8HTP3A64SkJRFxXNIS4GQn6wKQtAx4BLg9Il5u8tiz2l6zrKnIz4f8AFgkaUE6apv1T4y0qlHSAuDfAKuneYxj6fqkpEeoDTvMKdCKbrtpno+2/wxLgW21EfglYE2kQewGj9H2bdVAkXWf7DOenuOFwKk213EOSW+lFvxfjoi/mHp//c4gIh6X9DlJl0RE0x+i87BPHUlvAW6lNs7XrM+Supv/Gmj2DqFdNb1Dtf+DgKTLqO3NX2nQdRewMU1vBKY9Ym9DXYuA3cC9EfG/p+lX5vbaBWxIZ2OsoLatvlnfIQXLk8AtqamT2+p9wIsRMd7oTkkXSXr75DS1d0+dfj0VeT5K/RkW1f7B0z3AL0fEa036lLWtiqx7/d/aLcDXmu2w2iF9nrANOBQRf9Ckz89Mfu4g6Wpq2f7DaR+4k59SV/FC7QU/Tu1o+gTwRN19Q8C+BvN8gXRWCfAl4ADwHLUXwZJO1kVtPP0gtbMOvg38qyZ1/TSwFzicri/ucF3/Cfgx8Gzd5dIytleL5/Aj1M7WeAm4sa79cf7x7KTLqO0UxoD/AVzQodfaHwO/MaXtZ4HH6+r4TrocpDYE0unXf8Pno76udPsmameVvNzputLz8P2619Hnp9ZU5rZqtO7A/dR2TgAXptfNWHodXdbh7fMvqA0rPVe3jW4CfmPy9QV8qC4n9gH/vNXj+ucdzMwy5GEfM7MMOfzNzDLk8Dczy5DD38wsQw5/M7MMOfzNzDLk8Dczy9D/B9felEnnkVa8AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ax = pd.Series(np.log(np.abs(weightvalues)/sqrtsigmavalues + 1e-8)).hist(bins=500)\n",
    "#ax.set_xlim((0.00, 1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vanilla Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_adam(X, Y, X_test, Y_test,\n",
    "               num_epochs=10, learning_rate = 0.0001, dropout= 0.0, l2=0.0):\n",
    "    \n",
    "    # learning_rate = 0.001 ## good for ~ 10-20 epochs\n",
    "    # learning_rate = 0.0001 ## good for ~ 30-100 epochs\n",
    "    \n",
    "    \n",
    "    cv_accuracy_results = []\n",
    "    \n",
    "    tf.reset_default_graph()\n",
    "    tf.set_random_seed(1)\n",
    "    \n",
    "    inputs = tf.placeholder(tf.float32, [None, 784])\n",
    "    labels = tf.placeholder(tf.int32, [None, 10])\n",
    "    training = tf.placeholder(tf.bool, ())\n",
    "    input_d = 784\n",
    "    \n",
    "    with tf.variable_scope('nn'):\n",
    "        regularizer = tf.contrib.layers.l2_regularizer(scale=l2)\n",
    "        _hidden = tf.layers.dense(inputs, 400, \n",
    "                                  kernel_initializer=tf.initializers.random_normal(mean=0.0, stddev=0.1, seed=None),\n",
    "                                  bias_initializer=tf.initializers.random_normal(mean=0.0, stddev=0.1, seed=None),\n",
    "                                  kernel_regularizer=regularizer,\n",
    "                                  bias_regularizer=regularizer,\n",
    "                                  name='dense_l1')\n",
    "        hidden_d = tf.layers.dropout(_hidden, rate=dropout, training=training)\n",
    "        hidden = tf.nn.relu(hidden_d)\n",
    "        \n",
    "        _hidden = tf.layers.dense(hidden, 400, kernel_initializer=tf.initializers.random_normal(mean=0.0, stddev=0.1, seed=None),\n",
    "                                  bias_initializer=tf.initializers.random_normal(mean=0.0, stddev=0.1, seed=None),\n",
    "                                  kernel_regularizer=regularizer,\n",
    "                                  bias_regularizer=regularizer,\n",
    "                                  name='dense_l2')\n",
    "        hidden_d = tf.layers.dropout(_hidden, rate=dropout, training=training)\n",
    "        hidden = tf.nn.relu(hidden_d)\n",
    "        \n",
    "        logits = tf.layers.dense(hidden, 10, kernel_initializer=tf.initializers.random_normal(mean=0.0, stddev=0.1, seed=None),\n",
    "                                 bias_initializer=tf.initializers.random_normal(mean=0.0, stddev=0.1, seed=None),\n",
    "                                 name='dense_out')\n",
    "        out = tf.nn.softmax(logits)\n",
    "        \n",
    "    tmp = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=labels, logits=logits))\n",
    "    loss = tmp + tf.losses.get_regularization_loss()\n",
    "    opt = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "    \n",
    "    opt_train = opt.minimize(loss)\n",
    " \n",
    "    correct_prediction = tf.equal(tf.argmax(out,1), tf.argmax(labels,1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    \n",
    "    with tf.Session() as sess:\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "\n",
    "        for epoch in range(num_epochs):\n",
    "\n",
    "            i=0\n",
    "            for x, y in batches(X,Y):\n",
    "                sess.run(opt_train, {inputs: x, labels: y, training: True})\n",
    "                \n",
    "#                 if i <100:\n",
    "#                     print(sess.run(weights[3]))\n",
    "#                     xxx = opt.get_slot(weights[3], \"sigma\")\n",
    "#                     print(\"sigma: {}\".format(sess.run(xxx)))\n",
    "#                     print(\"m: {}\".format(sess.run(opt.get_slot(weights[3], \"m\"))))\n",
    "#                     print(\"v: {}\".format(sess.run(opt.get_slot(weights[3], \"v\"))))\n",
    "                    \n",
    "                    \n",
    "                i+=1\n",
    "                if i%1000==0:\n",
    "                    acc = sess.run(accuracy, {inputs: X_test,\n",
    "                                 labels: Y_test})\n",
    "                    cv_accuracy_results.append(acc)\n",
    "            acc, l = sess.run([accuracy, loss], {inputs: X_test,  labels: Y_test, training: False})       \n",
    "            cv_accuracy_results.append(acc)\n",
    "\n",
    "            print('Epoch: {:>2}: Validation loss: {:>3.5f}, Validation accuracy: {:>3.5f}'.format(epoch, l, acc))\n",
    "        \n",
    "        \n",
    "        weights = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=tf.get_variable_scope().name)\n",
    "        \n",
    "        # store flattened arrays:\n",
    "        weightvalues = np.hstack(np.array([sess.run(w).flatten() for w in weights]))\n",
    "                \n",
    "        # store weights:\n",
    "        _weights = [sess.run(w) for w in weights]\n",
    "        \n",
    "        def resetweights():\n",
    "            for v,_v in zip(weights, _weights):\n",
    "                v_t = state_ops.assign(v,tf.cast(_v, v.dtype))\n",
    "                sess.run(v_t)\n",
    "        \n",
    "        def pruning(remove_pct):\n",
    "            sorted_STN = np.sort(np.abs(weightvalues))\n",
    "            cuttoff = sorted_STN[int(remove_pct*len(sorted_STN))]\n",
    "\n",
    "            for v in weights:\n",
    "                mask = tf.greater_equal(tf.abs(v),cuttoff)\n",
    "                v_t = state_ops.assign(v,tf.multiply(v, tf.cast(mask, v.dtype)))\n",
    "                sess.run(v_t)\n",
    "            #print(sess.run(weights[-1]))  \n",
    "            \n",
    "            acc = sess.run(accuracy, {inputs: X_test, labels: Y_test, training:False})\n",
    "            print(\"%.2f, %s\" % (np.sum(sorted_STN<cuttoff)/len(sorted_STN),acc ))\n",
    "            return acc\n",
    "        \n",
    "        xs = np.append(0.1 * np.array(range(10)), [0.95, 0.96, 0.97, 0.98, 0.99, 0.992, 0.995, 0.997, 0.999])\n",
    "        ya = []\n",
    "        print(\"Pruning smallest elements (baseline):\")\n",
    "        for pct in xs:\n",
    "            ya.append(pruning(pct))\n",
    "        \n",
    "        resetweights()\n",
    "\n",
    "        return xs, ya, cv_accuracy_results, weightvalues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "runs=1\n",
    "xx = []\n",
    "yy = []\n",
    "accs = []\n",
    "ww = []\n",
    "for _ in range(runs):\n",
    "    x, ya, acc, weightvalues = train_adam(X_train, Y_train, X_test, Y_test, \n",
    "                                          num_epochs=100, learning_rate = 0.001, dropout= 0.25, l2=0.0)\n",
    "    xx.append(x)\n",
    "    yy.append(ya)\n",
    "    accs.append(acc)\n",
    "    ww.append(weightvalues)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.Series(1-np.array(accs[0])).plot(logy=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.DataFrame(pd.Series(x).rename('pruning_pct'))\n",
    "data['adam'] = 100*(1-np.array(ya))\n",
    "#data['our'] = 100*(1-np.array(yb))\n",
    "data = data.set_index('pruning_pct')\n",
    "data.plot(logy=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = pd.Series(weightvalues).hist(bins=500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}