Untitled.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "%matplotlib ipympl\n",
    "#%load_ext ipycache\n",
    "import matplotlib.pyplot as pyplot"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data preparation\n",
    "Here we'll generate syntetic image using PIL and system wide fonts. The make_img helper function generate.\n",
    "Since our network will only accept fixed size input, we hardcode a 105 x 105 size."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import PIL\n",
    "from PIL import Image, ImageFont, ImageDraw\n",
    "\n",
    "def make_img(text, font):\n",
    "  img = PIL.Image.new(\"RGB\", (105, 105))\n",
    "  draw = PIL.ImageDraw.Draw(img)\n",
    "  font = PIL.ImageFont.truetype(font, size=45)\n",
    "  draw.text((0, 0), text, font = font)\n",
    "  return img"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "While we're at it, let's see what kind of image it produces. Feel free to replace the true type font file according to your system path."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "010ab509bb434ffeb20bbc9286d08883",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FigureCanvasNbAgg()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x1c722577320>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "\n",
    "if sys.platform == \"win32\":\n",
    "    font_path = \"C:\\\\windows\\\\fonts\\\\arial.ttf\"\n",
    "elif sys.platform == \"linux\":\n",
    "    font_path = \"\"\n",
    "pyplot.imshow(make_img(\"test\", font_path))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There's obviously room for improvement..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training and validation dataset are passed through a Dataset interface. Such interface needs to implement 3 members functions, __init__(), __len__() and __get_item__(idx) which are rather self explanatory.\n",
    "Our implementation will create a random 4 letter text and pick a font and returns a text image along with the font index (data needs to be labeled by their class).\n",
    "\n",
    "By the way we use the name VFR for Visual Font Recognition."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-7-abd43d6520ef>, line 17)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  File \u001b[1;32m\"<ipython-input-7-abd43d6520ef>\"\u001b[1;36m, line \u001b[1;32m17\u001b[0m\n\u001b[1;33m    elif sys.plaform = \"linux\":\u001b[0m\n\u001b[1;37m                     ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "from string import ascii_lowercase\n",
    "import random\n",
    "from random import choice\n",
    "import os\n",
    "import torch.utils.data\n",
    "\n",
    "random.seed(0)    \n",
    "class CustomVFR(torch.utils.data.Dataset):\n",
    "  def get_word(self):\n",
    "    return ''.join([choice(ascii_lowercase) for _ in range(4)])\n",
    "  def __init__(self, size, transform = None):\n",
    "    self.transform = transform\n",
    "    self.fonts = []\n",
    "    self.size = size\n",
    "    if sys.platform == \"win32\":\n",
    "        font_directory = \"C:\\\\windows\\\\fonts\"\n",
    "    elif sys.plaform == \"linux\":\n",
    "        font_directory = \"\"\n",
    "    for path, dir, files in os.walk(font_directory):\n",
    "      for file in files:\n",
    "        if file.endswith(\".ttf\"):\n",
    "            self.fonts.append(file)\n",
    "    self.texts = [self.get_word() for _ in range(len(self.fonts) * self.size)]\n",
    "    self.cache = dict()\n",
    "\n",
    "  def __len__(self):\n",
    "    return len(self.fonts) * self.size\n",
    "\n",
    "  def __getitem__(self, i):\n",
    "    if (i in self.cache):\n",
    "        return self.cache[i]\n",
    "    idx = i % len(self.fonts)\n",
    "    word = self.texts[i]\n",
    "    img = make_img(word, self.fonts[idx])\n",
    "    if self.transform:\n",
    "      img = self.transform(img)\n",
    "    self.cache[i] = img, idx\n",
    "    return self.cache[i]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The network\n",
    "Now our network : it only needs to implement an __init__() and a forward(x) method and to inherit from the Module class.\n",
    "The x argument in the forward method is a torch Tensor object which will record gradient of their operator so that a backward operation is actually not needed.\n",
    "The Network structure comes from Adobe DeepFont paper : https://arxiv.org/abs/1507.03196"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "class DeepFont(torch.nn.Module):\n",
    "  def __init__(self):\n",
    "    super(DeepFont, self).__init__()\n",
    "    # unsupervisedly trained layers\n",
    "    self.conv_layer1 = torch.nn.Conv2d(1, 64, 9, stride=2)\n",
    "    self.lrn1 = torch.nn.LocalResponseNorm(2)\n",
    "    self.max_pooling1 = torch.nn.MaxPool2d(2, return_indices=True)\n",
    "    self.conv_layer2 = torch.nn.Conv2d(64, 128, 3, padding=1)\n",
    "    \n",
    "    self.deconv_layer1 = torch.nn.ConvTranspose2d(128, 64, 3, padding=1)\n",
    "    self.max_unpooling1 = torch.nn.MaxUnpool2d(2)\n",
    "    self.deconv_layer2 = torch.nn.ConvTranspose2d(64, 1, 11, stride=2)\n",
    "    \n",
    "    self.lrn2 = torch.nn.LocalResponseNorm(2)\n",
    "    self.max_pooling2 = torch.nn.MaxPool2d(2)\n",
    "\n",
    "    self.conv_layer3 = torch.nn.Conv2d(128, 256, 3, padding=1)\n",
    "    self.conv_layer4 = torch.nn.Conv2d(256, 256, 3, padding=1)\n",
    "    self.conv_layer5 = torch.nn.Conv2d(256, 256, 3, padding=1)\n",
    "    self.fc6 = torch.nn.Linear(256*12*12, 4096)\n",
    "    self.fc7 = torch.nn.Linear(4096, 2383)\n",
    "    self.fc8 = torch.nn.Linear(2383, 2383)\n",
    "    \n",
    "    self.unsupervised_learning = True\n",
    "    \n",
    "  def freeze_unsupervised(self):\n",
    "    self.unsupervised_learning = False\n",
    "    self.conv_layer1.requires_grad = False\n",
    "    self.lrn1.requires_grad = False\n",
    "    self.max_pooling1.requires_grad = False\n",
    "    self.conv_layer2.requires_grad = False\n",
    "\n",
    "  def forward(self, x):\n",
    "    x = self.conv_layer1(x)\n",
    "    x = F.relu(x)\n",
    "    x = self.lrn1(x)\n",
    "    x, indexes = self.max_pooling1(x)\n",
    "\n",
    "    x = self.conv_layer2(x)\n",
    "    x = F.relu(x)\n",
    "    \n",
    "    if self.unsupervised_learning:\n",
    "        x = self.deconv_layer1(x)\n",
    "        x = F.relu(x)\n",
    "        x = self.max_unpooling1(x, indexes)\n",
    "        x = self.deconv_layer2(x)\n",
    "        return x\n",
    "    else:\n",
    "        x = self.lrn2(x)\n",
    "        x = self.max_pooling2(x)\n",
    "        x = self.conv_layer3(x)\n",
    "        x = F.relu(x)\n",
    "        x = self.conv_layer4(x)\n",
    "        x = F.relu(x)\n",
    "        x = x.view(-1, 256*12*12)\n",
    "        x = F.dropout(x, training=self.training)\n",
    "        x = self.fc6(x)\n",
    "        x = F.relu(x)\n",
    "        x = F.dropout(x, training=self.training)\n",
    "        x = self.fc7(x)\n",
    "        x = F.relu(x)\n",
    "        x = self.fc8(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training & Validation\n",
    "Instanciate our dataset : we can use the transforms mechanism to alter our data before passing them to a network. In our case we need to convert PIL image to torch tensor ; we could also rotate, crop or whatever images to increase robustness (see: data augmentation).\n",
    "As usual we use separate set for training and validation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "trfrm = transforms.Compose( \n",
    "    [\n",
    "        transforms.ColorJitter(),\n",
    "        transforms.Grayscale(num_output_channels=1),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n",
    "    ])\n",
    "\n",
    "trainingset = CustomVFR(100, transform = trfrm)\n",
    "testset = CustomVFR(10, transform = trfrm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To ease dataset manipulation pytorch has a DataLoader object that can batch and shuffle data. Since pytorch network always take batch as input it's useful to rely on such object."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainingloader = torch.utils.data.DataLoader(trainingset, batch_size=128,\n",
    "                        shuffle=True)\n",
    "testloader = torch.utils.data.DataLoader(testset, batch_size=128,\n",
    "                        shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now the real part: we instanciate the network and train it using the cross entropy loss criterion and the Stochastic Gradient Descent optimisation algorithm."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "Res = DeepFont()\n",
    "Res.to(device)\n",
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(Res.parameters(), lr=0.001, momentum=0.9)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll use a small class to implement both progress bars (training can be long, it's better to have some feedback) and plot:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ProgressReport:\n",
    "    def __init__(self, training_set_count, figure_id, labels):\n",
    "        from ipywidgets import FloatProgress\n",
    "        from IPython import display\n",
    "\n",
    "        self.all_epoch_progress = FloatProgress(min=0, max=100, description='all epoch progress')\n",
    "        self.current_epoch_progress = FloatProgress(min=0, max=training_set_count, description='current epoch progress')\n",
    "        \n",
    "        display.display(self.all_epoch_progress)\n",
    "        display.display(self.current_epoch_progress)\n",
    "        \n",
    "        self.fig = fig = pyplot.figure(figure_id)\n",
    "        self.labels = labels\n",
    "        self.plots = []\n",
    "        for i, label in enumerate(labels):\n",
    "            ax = pyplot.subplot(len(labels), 1, i + 1)\n",
    "            ax.plot(range(100), range(100), color=label[1])\n",
    "            self.plots.append(ax)\n",
    "        self.epoch_data = []\n",
    "        \n",
    "    def mark_next_batch(self):\n",
    "        self.current_epoch_progress.value += 1\n",
    "        \n",
    "    def mark_epoch(self, value):\n",
    "          self.epoch_data.append(value)\n",
    "          self.current_epoch_progress.value = 0\n",
    "          self.all_epoch_progress.value += 1\n",
    "          for i, plt in enumerate(self.plots):\n",
    "              plt.lines[0].set_xdata(range(len(self.epoch_data)))\n",
    "              np_unsup_ep_loss = np.asarray([v[i] for v in self.epoch_data])\n",
    "              plt.lines[0].set_ydata(np_unsup_ep_loss)\n",
    "              plt.set_xlim(0, 100)\n",
    "              plt.set_ylim(np_unsup_ep_loss.min(), np_unsup_ep_loss.max() + 1)\n",
    "          pyplot.legend([plt.lines[0] for plt in self.plots], [label for label, _ in self.labels])\n",
    "          self.fig.canvas.draw()\n",
    "    @property\n",
    "    def data(self):\n",
    "        return self.epoch_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Unsupervised learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Stacked Convolutional Auto Encoder training part  basically takes the first half of the network, plug it into some kind of reverse network and then train the whole new network to be as close as possible from an identity transformation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f2b9270fe1e04cdc951942646b8755b1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, description='all epoch progress')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c49f9909890e4d7eb1cfb455d6a2855d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, description='current epoch progress', max=96.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8fb3b8e65dba4f559e67f3416164ef70",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FigureCanvasNbAgg()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#%%cache Untitled.pkl unsupervised_epoch_losses\n",
    "import numpy as np\n",
    "\n",
    "unsupervised_criterion = torch.nn.MSELoss()\n",
    "PR = ProgressReport(len(trainingloader), 2, [('loss', 'r')])\n",
    "#pyplot.yscale('log')\n",
    "for epoch in range(100):\n",
    "  epoch_loss = 0\n",
    "  for imgs, _ in trainingloader:\n",
    "    imgs = imgs.to(device)\n",
    "    optimizer.zero_grad()\n",
    "    outputs = Res(imgs)\n",
    "\n",
    "    loss = unsupervised_criterion(outputs.type_as(imgs), imgs)\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    epoch_loss += loss.item()\n",
    "    PR.mark_next_batch()\n",
    "  PR.mark_epoch([epoch_loss])\n",
    "unsupervised_epoch_losses = PR.data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll use 100 epoch ; this means we'll pass training data 100 times in the network.\n",
    "For every epoch we run the network on our validation data to see how accurate the prediction is.\n",
    "The two value are plotted below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa4b2f46446b468dad8fde1074a95079",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, description='all epoch progress')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9c62b5818d1b456d9f6d90a64eb5a0fb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, description='current epoch progress', max=106.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "313608f7d4cc4f259ad029ede74c6ae5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FigureCanvasNbAgg()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#%%cache Untitled.pkl unsupervised_epoch_losses\n",
    "Res.freeze_unsupervised()\n",
    "\n",
    "PR = ProgressReport(len(trainingloader) + len(testloader), 3, [('training', 'b'), ('validation', 'r')])\n",
    "epoch_losses = []\n",
    "Res.train()\n",
    "for epoch in range(100):\n",
    "  epoch_loss = 0\n",
    "  correct = 0\n",
    "  all = 0\n",
    "  for imgs, fonts in trainingloader:\n",
    "    imgs, fontsgpu = imgs.to(device), fonts.to(device)\n",
    "    optimizer.zero_grad()\n",
    "    outputs = Res(imgs)\n",
    "    \n",
    "    _, idx = torch.max(outputs, 1)\n",
    "    tmp = (idx.cpu() == fonts).sum().item()\n",
    "    correct += tmp\n",
    "    all += len(fonts)\n",
    "\n",
    "    loss = criterion(outputs, fontsgpu)\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    epoch_loss += loss.item()\n",
    "    PR.mark_next_batch()\n",
    "  training_accuracy = correct/all\n",
    "  correct = 0\n",
    "  all = 0\n",
    "  Res.eval()\n",
    "  with torch.no_grad():\n",
    "    for imgs, fonts in testloader:\n",
    "      imgs = imgs.to(device)\n",
    "      outputs = Res(imgs)\n",
    "      _, idx = torch.max(outputs, 1)\n",
    "      tmp = (idx.cpu() == fonts).sum().item()\n",
    "      correct += tmp\n",
    "      all += len(fonts)\n",
    "      PR.mark_next_batch()\n",
    "  epoch_losses.append(epoch_loss)\n",
    "  test_accuracy = correct/all\n",
    "  PR.mark_epoch((training_accuracy, test_accuracy))\n",
    "epoch_accuracy = PR.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}