train_search_nb.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "\n",
    "from fastai import *\n",
    "from fastai.text import *\n",
    "from fastai.callbacks import CSVLogger, SaveModelCallback\n",
    "\n",
    "import torch.backends.cudnn as cudnn\n",
    "\n",
    "from train_search import DartsRnnSearch, ArchParamUpdate, PrintGenotype\n",
    "from darts_callbacks import HidInit, Regu, SaveModel, ResumeModel, GcCol "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# random seed for reproducibility.\n",
    "seed = 135\n",
    "random.seed(seed)\n",
    "np.random.seed(seed)\n",
    "torch.manual_seed(seed)\n",
    "\n",
    "gpu = 0\n",
    "torch.cuda.set_device(gpu)\n",
    "cudnn.benchmark = True\n",
    "cudnn.enabled=True\n",
    "torch.backends.cudnn.deterministic = True\n",
    "torch.cuda.manual_seed_all(seed)\n",
    "\n",
    "bptt = 35\n",
    "bs_train, bs_val = 256, 64\n",
    "dat = load_data('data', 'penn_db', bs=bs_train, bptt=bptt)\n",
    "dat.valid_dl.batch_size = bs_val\n",
    "search_dat = list(iter(load_data('data', 'penn_db', bs=bs_train, bptt = bptt).valid_dl))\n",
    "\n",
    "arch_lr = 3e-3\n",
    "arch_wdecay = 1e-3\n",
    "vocab_sz = len(dat.train_ds.x.vocab.itos)\n",
    "emb_sz = 300\n",
    "hid_sz = 300\n",
    "wdecay = 5e-7\n",
    "dropout = 0.75\n",
    "dropouth = 0.25\n",
    "dropoutx = 0.75\n",
    "dropouti = 0.2\n",
    "dropoute = 0.\n",
    "clip = 0.25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "csv_name = 'train_search'\n",
    "model_name = 'train_search'\n",
    "# resume_model = 'train_search'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total params: 4810000\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 1), ('identity', 1), ('relu', 3), ('sigmoid', 4), ('sigmoid', 3), ('identity', 6), ('identity', 1)], concat=range(1, 9))\n"
     ]
    }
   ],
   "source": [
    "learn = Learner(dat, DartsRnnSearch(emb_sz = emb_sz, vocab_sz = vocab_sz,\n",
    "                                    ninp = emb_sz, nhid = hid_sz, \n",
    "                                    dropout = dropout, dropouth = dropouth, dropoutx = dropoutx,\n",
    "                                    dropouti = dropouti, dropoute = dropoute,\n",
    "                                    bs_train = bs_train, bs_val = bs_val),\n",
    "                opt_func = torch.optim.SGD,\n",
    "                callback_fns = [\n",
    "                    HidInit,\n",
    "                    partial(ArchParamUpdate, search_dat=search_dat,\n",
    "                            arch_lr=arch_lr, arch_wdecay=arch_wdecay, wdecay=wdecay),\n",
    "                    Regu,\n",
    "                    PrintGenotype,\n",
    "                    partial(GradientClipping, clip=clip),\n",
    "                    partial(CSVLogger, filename = csv_name, append = True),\n",
    "                    GcCol,\n",
    "#                     partial(ResumeModel, name = resume_model) \n",
    "                ], \n",
    "                wd = wdecay\n",
    "                )\n",
    "\n",
    "total_params = sum(x.nelement() for x in learn.model.parameters())\n",
    "print('Total params:', total_params)\n",
    "print(learn.model.genotype_parse())\n",
    "# learn.data.valid_dl=None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 4:21:05 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>9.377739</td>\n",
       "      <td>8.914400</td>\n",
       "      <td>05:15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>9.024400</td>\n",
       "      <td>7.874600</td>\n",
       "      <td>05:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>8.465348</td>\n",
       "      <td>7.455300</td>\n",
       "      <td>05:16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>8.251266</td>\n",
       "      <td>7.209497</td>\n",
       "      <td>05:15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>8.137778</td>\n",
       "      <td>7.105574</td>\n",
       "      <td>05:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>8.029912</td>\n",
       "      <td>6.999118</td>\n",
       "      <td>05:15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>7.972732</td>\n",
       "      <td>6.931962</td>\n",
       "      <td>05:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>7.771575</td>\n",
       "      <td>6.910496</td>\n",
       "      <td>05:16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>7.630065</td>\n",
       "      <td>6.853231</td>\n",
       "      <td>05:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>7.517681</td>\n",
       "      <td>6.790563</td>\n",
       "      <td>05:15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>7.386724</td>\n",
       "      <td>6.772378</td>\n",
       "      <td>05:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>7.236539</td>\n",
       "      <td>6.759993</td>\n",
       "      <td>05:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>7.063982</td>\n",
       "      <td>6.703610</td>\n",
       "      <td>05:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>6.905414</td>\n",
       "      <td>6.662308</td>\n",
       "      <td>05:09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>6.708851</td>\n",
       "      <td>6.653984</td>\n",
       "      <td>05:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>6.539793</td>\n",
       "      <td>6.653328</td>\n",
       "      <td>05:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>6.402159</td>\n",
       "      <td>6.616840</td>\n",
       "      <td>05:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>6.250990</td>\n",
       "      <td>6.605050</td>\n",
       "      <td>05:16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>6.140637</td>\n",
       "      <td>6.613458</td>\n",
       "      <td>05:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>6.026720</td>\n",
       "      <td>6.599876</td>\n",
       "      <td>05:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>5.945973</td>\n",
       "      <td>6.587843</td>\n",
       "      <td>05:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21</td>\n",
       "      <td>5.857794</td>\n",
       "      <td>6.579211</td>\n",
       "      <td>05:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22</td>\n",
       "      <td>5.771217</td>\n",
       "      <td>6.584734</td>\n",
       "      <td>05:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23</td>\n",
       "      <td>5.714971</td>\n",
       "      <td>6.569280</td>\n",
       "      <td>05:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24</td>\n",
       "      <td>5.660535</td>\n",
       "      <td>6.567819</td>\n",
       "      <td>05:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25</td>\n",
       "      <td>5.617890</td>\n",
       "      <td>6.582723</td>\n",
       "      <td>05:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26</td>\n",
       "      <td>5.579669</td>\n",
       "      <td>6.570678</td>\n",
       "      <td>05:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>27</td>\n",
       "      <td>5.551519</td>\n",
       "      <td>6.581318</td>\n",
       "      <td>05:09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28</td>\n",
       "      <td>5.528355</td>\n",
       "      <td>6.565953</td>\n",
       "      <td>05:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29</td>\n",
       "      <td>5.496869</td>\n",
       "      <td>6.573961</td>\n",
       "      <td>05:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>5.477722</td>\n",
       "      <td>6.563119</td>\n",
       "      <td>05:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>31</td>\n",
       "      <td>5.459666</td>\n",
       "      <td>6.568295</td>\n",
       "      <td>05:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32</td>\n",
       "      <td>5.440758</td>\n",
       "      <td>6.562510</td>\n",
       "      <td>05:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>33</td>\n",
       "      <td>5.418176</td>\n",
       "      <td>6.566048</td>\n",
       "      <td>05:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>34</td>\n",
       "      <td>5.401654</td>\n",
       "      <td>6.563069</td>\n",
       "      <td>05:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>35</td>\n",
       "      <td>5.380398</td>\n",
       "      <td>6.576956</td>\n",
       "      <td>05:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>36</td>\n",
       "      <td>5.363576</td>\n",
       "      <td>6.564011</td>\n",
       "      <td>05:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>37</td>\n",
       "      <td>5.353090</td>\n",
       "      <td>6.572284</td>\n",
       "      <td>05:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>38</td>\n",
       "      <td>5.335605</td>\n",
       "      <td>6.563518</td>\n",
       "      <td>05:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>39</td>\n",
       "      <td>5.327987</td>\n",
       "      <td>6.566296</td>\n",
       "      <td>05:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40</td>\n",
       "      <td>5.317882</td>\n",
       "      <td>6.569047</td>\n",
       "      <td>05:15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>41</td>\n",
       "      <td>5.310230</td>\n",
       "      <td>6.571513</td>\n",
       "      <td>05:15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>42</td>\n",
       "      <td>5.296213</td>\n",
       "      <td>6.565657</td>\n",
       "      <td>05:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>43</td>\n",
       "      <td>5.285232</td>\n",
       "      <td>6.565644</td>\n",
       "      <td>05:16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>44</td>\n",
       "      <td>5.273549</td>\n",
       "      <td>6.572789</td>\n",
       "      <td>05:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>45</td>\n",
       "      <td>5.266336</td>\n",
       "      <td>6.559350</td>\n",
       "      <td>05:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>46</td>\n",
       "      <td>5.260991</td>\n",
       "      <td>6.564308</td>\n",
       "      <td>05:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>47</td>\n",
       "      <td>5.254736</td>\n",
       "      <td>6.575763</td>\n",
       "      <td>05:18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>48</td>\n",
       "      <td>5.246768</td>\n",
       "      <td>6.559757</td>\n",
       "      <td>05:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>49</td>\n",
       "      <td>5.242108</td>\n",
       "      <td>6.566053</td>\n",
       "      <td>05:14</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Genotype(recurrent=[('identity', 0), ('identity', 0), ('relu', 2), ('sigmoid', 2), ('sigmoid', 2), ('tanh', 2), ('relu', 2), ('relu', 5)], concat=range(1, 9))\n",
      "Better model found at epoch 0 with val_loss value: 8.914400100708008.\n",
      "Genotype(recurrent=[('relu', 0), ('tanh', 1), ('identity', 2), ('relu', 3), ('sigmoid', 0), ('tanh', 2), ('tanh', 5), ('relu', 4)], concat=range(1, 9))\n",
      "Better model found at epoch 1 with val_loss value: 7.874600410461426.\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('tanh', 2), ('identity', 0), ('sigmoid', 1), ('tanh', 2), ('identity', 5), ('relu', 4)], concat=range(1, 9))\n",
      "Better model found at epoch 2 with val_loss value: 7.455300331115723.\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('tanh', 2), ('identity', 2), ('tanh', 2), ('tanh', 2), ('tanh', 4), ('sigmoid', 2)], concat=range(1, 9))\n",
      "Better model found at epoch 3 with val_loss value: 7.209496974945068.\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('tanh', 2), ('identity', 2), ('tanh', 2), ('tanh', 2), ('tanh', 4), ('sigmoid', 2)], concat=range(1, 9))\n",
      "Better model found at epoch 4 with val_loss value: 7.105574131011963.\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('tanh', 2), ('identity', 2), ('tanh', 2), ('tanh', 2), ('identity', 5), ('sigmoid', 2)], concat=range(1, 9))\n",
      "Better model found at epoch 5 with val_loss value: 6.999118328094482.\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('tanh', 2), ('identity', 2), ('tanh', 2), ('tanh', 2), ('tanh', 4), ('sigmoid', 2)], concat=range(1, 9))\n",
      "Better model found at epoch 6 with val_loss value: 6.931961536407471.\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('tanh', 2), ('identity', 2), ('tanh', 2), ('tanh', 2), ('identity', 5), ('sigmoid', 2)], concat=range(1, 9))\n",
      "Better model found at epoch 7 with val_loss value: 6.910495758056641.\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('tanh', 2), ('identity', 2), ('tanh', 0), ('tanh', 2), ('identity', 5), ('sigmoid', 2)], concat=range(1, 9))\n",
      "Better model found at epoch 8 with val_loss value: 6.853230953216553.\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('tanh', 2), ('identity', 2), ('tanh', 0), ('tanh', 2), ('identity', 2), ('sigmoid', 2)], concat=range(1, 9))\n",
      "Better model found at epoch 9 with val_loss value: 6.790562629699707.\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('tanh', 2), ('identity', 2), ('tanh', 2), ('tanh', 2), ('identity', 5), ('sigmoid', 2)], concat=range(1, 9))\n",
      "Better model found at epoch 10 with val_loss value: 6.772377967834473.\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('tanh', 2), ('identity', 2), ('tanh', 2), ('tanh', 2), ('identity', 2), ('sigmoid', 2)], concat=range(1, 9))\n",
      "Better model found at epoch 11 with val_loss value: 6.759993076324463.\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('tanh', 2), ('identity', 2), ('identity', 4), ('tanh', 2), ('identity', 2), ('sigmoid', 2)], concat=range(1, 9))\n",
      "Better model found at epoch 12 with val_loss value: 6.703609943389893.\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('tanh', 2), ('identity', 2), ('identity', 0), ('tanh', 2), ('identity', 2), ('sigmoid', 2)], concat=range(1, 9))\n",
      "Better model found at epoch 13 with val_loss value: 6.6623077392578125.\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('tanh', 2), ('identity', 2), ('identity', 4), ('tanh', 2), ('identity', 2), ('relu', 7)], concat=range(1, 9))\n",
      "Better model found at epoch 14 with val_loss value: 6.653984069824219.\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('tanh', 2), ('identity', 2), ('identity', 4), ('tanh', 2), ('tanh', 5), ('relu', 7)], concat=range(1, 9))\n",
      "Better model found at epoch 15 with val_loss value: 6.653327941894531.\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('relu', 2), ('identity', 2), ('identity', 4), ('tanh', 2), ('identity', 2), ('relu', 7)], concat=range(1, 9))\n",
      "Better model found at epoch 16 with val_loss value: 6.616840362548828.\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('identity', 2), ('identity', 2), ('identity', 4), ('sigmoid', 0), ('identity', 2), ('relu', 7)], concat=range(1, 9))\n",
      "Better model found at epoch 17 with val_loss value: 6.605050086975098.\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('identity', 2), ('identity', 1), ('identity', 4), ('identity', 2), ('identity', 3), ('relu', 7)], concat=range(1, 9))\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('identity', 2), ('identity', 2), ('identity', 0), ('tanh', 5), ('identity', 3), ('relu', 7)], concat=range(1, 9))\n",
      "Better model found at epoch 19 with val_loss value: 6.599876403808594.\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('identity', 2), ('identity', 2), ('identity', 0), ('identity', 2), ('identity', 3), ('relu', 7)], concat=range(1, 9))\n",
      "Better model found at epoch 20 with val_loss value: 6.58784294128418.\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('identity', 2), ('identity', 2), ('identity', 0), ('tanh', 5), ('identity', 3), ('relu', 7)], concat=range(1, 9))\n",
      "Better model found at epoch 21 with val_loss value: 6.579211235046387.\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('relu', 2), ('identity', 2), ('identity', 0), ('relu', 4), ('identity', 3), ('relu', 7)], concat=range(1, 9))\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('identity', 2), ('identity', 2), ('identity', 0), ('relu', 4), ('tanh', 5), ('identity', 1)], concat=range(1, 9))\n",
      "Better model found at epoch 23 with val_loss value: 6.569279670715332.\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('identity', 2), ('identity', 2), ('identity', 0), ('relu', 4), ('identity', 3), ('identity', 1)], concat=range(1, 9))\n",
      "Better model found at epoch 24 with val_loss value: 6.567819118499756.\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('identity', 2), ('identity', 2), ('identity', 0), ('relu', 4), ('tanh', 5), ('identity', 1)], concat=range(1, 9))\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('identity', 2), ('identity', 2), ('identity', 0), ('relu', 4), ('tanh', 5), ('identity', 1)], concat=range(1, 9))\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('identity', 2), ('identity', 2), ('identity', 0), ('relu', 4), ('tanh', 5), ('identity', 1)], concat=range(1, 9))\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('identity', 2), ('identity', 2), ('identity', 0), ('identity', 0), ('tanh', 5), ('identity', 1)], concat=range(1, 9))\n",
      "Better model found at epoch 28 with val_loss value: 6.565952777862549.\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('identity', 2), ('identity', 2), ('identity', 0), ('identity', 2), ('tanh', 5), ('identity', 4)], concat=range(1, 9))\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('relu', 2), ('identity', 2), ('identity', 0), ('relu', 4), ('tanh', 5), ('relu', 5)], concat=range(1, 9))\n",
      "Better model found at epoch 30 with val_loss value: 6.563118934631348.\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('identity', 2), ('identity', 2), ('identity', 0), ('relu', 4), ('tanh', 5), ('relu', 5)], concat=range(1, 9))\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('relu', 2), ('identity', 2), ('identity', 0), ('relu', 4), ('tanh', 5), ('identity', 1)], concat=range(1, 9))\n",
      "Better model found at epoch 32 with val_loss value: 6.562509536743164.\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('identity', 2), ('identity', 2), ('identity', 0), ('relu', 4), ('tanh', 5), ('identity', 1)], concat=range(1, 9))\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('relu', 2), ('identity', 2), ('identity', 0), ('relu', 4), ('tanh', 5), ('relu', 5)], concat=range(1, 9))\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('relu', 2), ('identity', 2), ('identity', 0), ('identity', 0), ('tanh', 5), ('identity', 1)], concat=range(1, 9))\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('relu', 2), ('identity', 2), ('identity', 0), ('identity', 0), ('tanh', 5), ('relu', 5)], concat=range(1, 9))\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('relu', 2), ('identity', 2), ('identity', 0), ('identity', 0), ('tanh', 5), ('relu', 5)], concat=range(1, 9))\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('relu', 2), ('identity', 2), ('identity', 0), ('identity', 0), ('tanh', 5), ('relu', 5)], concat=range(1, 9))\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('relu', 2), ('identity', 2), ('identity', 0), ('identity', 0), ('tanh', 5), ('sigmoid', 2)], concat=range(1, 9))\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('relu', 2), ('identity', 2), ('identity', 0), ('identity', 0), ('tanh', 5), ('identity', 4)], concat=range(1, 9))\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('identity', 2), ('identity', 2), ('identity', 0), ('identity', 0), ('tanh', 5), ('identity', 4)], concat=range(1, 9))\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('identity', 2), ('identity', 2), ('identity', 0), ('identity', 0), ('tanh', 5), ('identity', 4)], concat=range(1, 9))\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('relu', 2), ('identity', 2), ('identity', 4), ('relu', 4), ('tanh', 5), ('relu', 5)], concat=range(1, 9))\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('identity', 2), ('identity', 2), ('identity', 0), ('relu', 4), ('tanh', 5), ('relu', 5)], concat=range(1, 9))\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('identity', 2), ('identity', 2), ('identity', 4), ('relu', 4), ('tanh', 5), ('relu', 5)], concat=range(1, 9))\n",
      "Better model found at epoch 45 with val_loss value: 6.55935001373291.\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('identity', 2), ('identity', 2), ('identity', 4), ('relu', 4), ('tanh', 5), ('relu', 5)], concat=range(1, 9))\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('relu', 2), ('identity', 2), ('identity', 4), ('relu', 4), ('tanh', 5), ('relu', 5)], concat=range(1, 9))\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('relu', 2), ('identity', 2), ('identity', 4), ('relu', 4), ('tanh', 5), ('relu', 5)], concat=range(1, 9))\n",
      "Genotype(recurrent=[('sigmoid', 0), ('tanh', 0), ('relu', 2), ('identity', 2), ('identity', 4), ('relu', 4), ('tanh', 5), ('relu', 5)], concat=range(1, 9))\n"
     ]
    }
   ],
   "source": [
    "learn.fit(50, 20, callbacks=[\n",
    "#     SaveModel(learn, gap = 1, name=model_name),\n",
    "    SaveModelCallback(learn, name=model_name) # save on improvement\n",
    "                           ])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}