face_net.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "face_net.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "mPWMCi_tCiix",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "8dc0a94b-aacf-4fa2-8708-d67ed5c33058"
      },
      "source": [
        "cd drive/My\\ Drive/facenet"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/My Drive/facenet\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u44YDz__CnUJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "9d821d60-dc72-4000-c18d-f30a782c9cc4"
      },
      "source": [
        "!ls"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "lfw  lfw.tgz  siamese_xception.h5\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NnePdNuk4oiu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tensorflow.keras.utils import Sequence, multi_gpu_model\n",
        "from tensorflow.keras.layers import Input, Dense, LeakyReLU, Concatenate, Lambda, BatchNormalization, GlobalAveragePooling2D\n",
        "from tensorflow.keras.applications.xception import Xception, preprocess_input\n",
        "from tensorflow.keras import backend as K\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.preprocessing import image\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "from tensorflow.keras.callbacks import ModelCheckpoint\n",
        "import tensorflow as tf"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u9-6bfEJBMvn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "class TripletImageLoader(tf.keras.utils.Sequence):\n",
        "    def __init__(self, df, preprocess_function, img_shape, batchSize = 16, flip=False):\n",
        "        self.files = df['image'].values\n",
        "        self.batchSize = batchSize\n",
        "        self.y = df['label'].values\n",
        "        self.N = len(self.y)\n",
        "        self.shape = img_shape\n",
        "        self.function = preprocess_function\n",
        "        self.flip = flip\n",
        "        \n",
        "    def load_image(self, file):\n",
        "        img = image.load_img( '{}'.format(file),target_size=(self.shape[0],self.shape[1]))\n",
        "        img = image.img_to_array(img)\n",
        "        img = self.function(img)\n",
        "        # img = tf.keras.backend.resize_images(img,self.shape[1],self.shape[0],data_format='channels_last')\n",
        "        if self.flip:\n",
        "            if np.random.randint(0, 2): # do flippings in 50% of the time\n",
        "                img = img[:, ::-1, :]\n",
        "        return img\n",
        "\n",
        "    #gets the number of batches this generator returns\n",
        "    def __len__(self):\n",
        "        l,rem = divmod(self.N, self.batchSize)\n",
        "        return (l + (1 if rem > 0 else 0))\n",
        "    \n",
        "    #shuffles data on epoch end\n",
        "    def on_epoch_end(self):\n",
        "        a = np.arange(len(self.y))\n",
        "        np.random.shuffle(a)\n",
        "        self.files = self.files[a]\n",
        "        self.y = self.y[a]\n",
        "        \n",
        "    #gets a batch with index = i\n",
        "    def __getitem__(self, i):\n",
        "        start = i*self.batchSize\n",
        "        stop  = np.min([(i+1)*self.batchSize, self.N]) # clip stop index to be <= N\n",
        "        # Memory preallocation\n",
        "        ANCHOR = np.zeros( (stop-start,) + self.shape + (3,) )\n",
        "        POSITIVE = np.zeros( (stop-start,) + self.shape + (3,) )\n",
        "        NEGATIVE = np.zeros( (stop-start,) + self.shape + (3,) )\n",
        "        ancor_labels = self.y[start:stop]\n",
        "        ancor_images = self.files[start:stop]\n",
        "        pos_images = []\n",
        "        neg_images = []\n",
        "        for k, label in enumerate(ancor_labels):\n",
        "            pos_idx = np.where(self.y==label)[0]\n",
        "            neg_idx = np.where(self.y!=label)[0]\n",
        "            neg_images.append(self.files[np.random.choice(neg_idx)])\n",
        "            pos_idx_hat = pos_idx[(pos_idx<start) | (pos_idx>stop)]\n",
        "            if len(pos_idx_hat):\n",
        "                pos_images.append(self.files[np.random.choice(pos_idx_hat)])\n",
        "            else:\n",
        "                # positive examples are within the batch or just 1 example in dataset\n",
        "                pos_images.append(self.files[np.random.choice(pos_idx)])\n",
        "                \n",
        "        for k, (a, p, n) in enumerate(zip(ancor_images, pos_images, neg_images)):\n",
        "            ANCHOR[k] = self.load_image(a)\n",
        "            POSITIVE[k] = self.load_image(p)\n",
        "            NEGATIVE[k] = self.load_image(n)\n",
        "            \n",
        "        return [ANCHOR, POSITIVE, NEGATIVE], np.empty(k+1) # we don't need labels so we reutrn dummy label (Keras requierments)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fqeHi1bN40QQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "xception = Xception(include_top=False, weights=None, input_shape = (218,178) + (3,))\n",
        "output = GlobalAveragePooling2D()(xception.output)\n",
        "base_model = Model(xception.input, output)\n",
        "\n",
        "PARALLEL = False\n",
        "\n",
        "def embedder(conv_feat_size):\n",
        "    '''\n",
        "    Takes the output of the conv feature extractor and yields the embeddings\n",
        "    '''\n",
        "    input = Input((conv_feat_size,), name = 'input')\n",
        "    normalize = Lambda(lambda x: K.l2_normalize(x, axis=-1), name='normalize')\n",
        "    x = Dense(512)(input)\n",
        "    x = LeakyReLU(alpha=0.1)(x)\n",
        "    x = Dense(128)(x)\n",
        "    x = normalize(x)\n",
        "    model = Model(input, x)\n",
        "    return model\n",
        "    \n",
        "def get_siamese_model(base_model):\n",
        "    \n",
        "    inp_shape = K.int_shape(base_model.input)[1:]\n",
        "    conv_feat_size = K.int_shape(base_model.output)[-1]\n",
        "    \n",
        "    input_a = Input( inp_shape,  name='anchor')\n",
        "    input_p = Input( inp_shape,  name='positive')\n",
        "    input_n = Input( inp_shape,  name='negative')\n",
        "    emb_model = embedder(conv_feat_size)\n",
        "    output_a = emb_model(base_model(input_a))\n",
        "    output_p = emb_model(base_model(input_p))\n",
        "    output_n = emb_model(base_model(input_n))\n",
        "    \n",
        "    merged_vector = Concatenate(axis=-1)([output_a, output_p, output_n])\n",
        "    model = Model(inputs=[input_a, input_p, input_n],\n",
        "                  outputs=merged_vector)\n",
        "\n",
        "    return model\n",
        "\n",
        "model = get_siamese_model(base_model)\n",
        "# model.load_weights('siamese_xception.h5')\n",
        "if PARALLEL:\n",
        "    parallel_model = multi_gpu_model(model, 1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pEFm7tMTA425",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def triplet_loss(y_true, y_pred, cosine = True, alpha = 0.2):\n",
        "    embedding_size = K.int_shape(y_pred)[-1] // 3\n",
        "    ind = int(embedding_size * 2)\n",
        "    a_pred = y_pred[:, :embedding_size]\n",
        "    p_pred = y_pred[:, embedding_size:ind]\n",
        "    n_pred = y_pred[:, ind:]\n",
        "    if cosine:\n",
        "        positive_distance = 1 - K.sum((a_pred * p_pred), axis=-1)\n",
        "        negative_distance = 1 - K.sum((a_pred * n_pred), axis=-1)\n",
        "    else:\n",
        "        positive_distance = K.sqrt(K.sum(K.square(a_pred - p_pred), axis=-1))\n",
        "        negative_distance = K.sqrt(K.sum(K.square(a_pred - n_pred), axis=-1))\n",
        "    loss = K.maximum(0.0, positive_distance - negative_distance + alpha)\n",
        "    return loss"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "02BnCpyiBDcw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.compile(Adam(lr = 0.0001), loss = triplet_loss)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GrjkvjM4BF3o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "checkpoint = ModelCheckpoint('siamese_xception.h5', monitor='val_loss', \n",
        "                             verbose=1, save_best_only=True, save_weights_only=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KRJ_k05MBVCM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "idx = list()\n",
        "values = list()\n",
        "for r , _ , f in os.walk(\"lfw\"):\n",
        "    if r not in idx:\n",
        "        idx.append(r)\n",
        "    id = idx.index(r)\n",
        "    for _f in f:\n",
        "        path = os.path.join(r,_f)\n",
        "        values.append([path,id])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pt5hRJrUBqeV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2b7rUejPB44X",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#!wget http://vis-www.cs.umass.edu/lfw/lfw.tgz\n",
        "# !tar -xvf lfw.tgz"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MOXLsluNB3l7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data_set = pd.DataFrame(values,columns=['image','label'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r58DdBcjCaxF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "145a8b87-b120-4287-f15a-61a33c0781cb"
      },
      "source": [
        "data_set.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>image</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>lfw/Darko_Milicic/Darko_Milicic_0001.jpg</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>lfw/Paul_Walker/Paul_Walker_0001.jpg</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>lfw/Gordon_Brown/Gordon_Brown_0001.jpg</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>lfw/Gordon_Brown/Gordon_Brown_0002.jpg</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>lfw/Gordon_Brown/Gordon_Brown_0003.jpg</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                      image  label\n",
              "0  lfw/Darko_Milicic/Darko_Milicic_0001.jpg      1\n",
              "1      lfw/Paul_Walker/Paul_Walker_0001.jpg      2\n",
              "2    lfw/Gordon_Brown/Gordon_Brown_0001.jpg      3\n",
              "3    lfw/Gordon_Brown/Gordon_Brown_0002.jpg      3\n",
              "4    lfw/Gordon_Brown/Gordon_Brown_0003.jpg      3"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2cMS8sWdCUR1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "  train = data_set.head(int(len(data_set)*(0.8)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tj-ckn7XCWpU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "val = data_set.iloc[max(train.index):]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "As5AbZPDBH-L",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_gen = TripletImageLoader(train, preprocess_input, (218,178), 8)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hizzXI8QB0IX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "val_gen = TripletImageLoader(val, preprocess_input, (218,178), 8)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jolrXYpWDs2F",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "es = tf.keras.callbacks.EarlyStopping(\n",
        "    monitor='val_loss', patience=2, verbose=1, mode='auto',restore_best_weights=True\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eriIlxllDu0f",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "1de5de4c-fe0a-4a6d-9c1b-7cafdf9592fc"
      },
      "source": [
        "model.fit(train_gen,epochs=20, steps_per_epoch=len(train_gen) ,validation_data= val_gen,callbacks=[checkpoint,es],validation_steps=len(val_gen))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/20\n",
            "  28/1899 [..............................] - ETA: 3:22:07 - loss: 0.1575"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "021h2AKuDxCf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tensorflow.keras.models import Model, load_model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e_49XJpn7T6q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.load_weights('siamese_xception.h5')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IlLVY8Ms7XCq",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 495
        },
        "outputId": "975a6f34-cfbc-4b24-bb3f-5ca2b8f1bac0"
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"functional_5\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "anchor (InputLayer)             [(None, 218, 178, 3) 0                                            \n",
            "__________________________________________________________________________________________________\n",
            "positive (InputLayer)           [(None, 218, 178, 3) 0                                            \n",
            "__________________________________________________________________________________________________\n",
            "negative (InputLayer)           [(None, 218, 178, 3) 0                                            \n",
            "__________________________________________________________________________________________________\n",
            "functional_1 (Functional)       (None, 2048)         20861480    anchor[0][0]                     \n",
            "                                                                 positive[0][0]                   \n",
            "                                                                 negative[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "functional_3 (Functional)       (None, 128)          1114752     functional_1[0][0]               \n",
            "                                                                 functional_1[1][0]               \n",
            "                                                                 functional_1[2][0]               \n",
            "__________________________________________________________________________________________________\n",
            "concatenate (Concatenate)       (None, 384)          0           functional_3[0][0]               \n",
            "                                                                 functional_3[1][0]               \n",
            "                                                                 functional_3[2][0]               \n",
            "==================================================================================================\n",
            "Total params: 21,976,232\n",
            "Trainable params: 21,921,704\n",
            "Non-trainable params: 54,528\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f8e2CXg97avb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "img = tf.keras.preprocessing.image.load_img('{}'.format(values[0][0])).resize((178,218))\n",
        "img = image.img_to_array(img).astype('uint8')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NjOnxpzL_R6K",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "inp = model.input[0]\n",
        "base_model = model.layers[3]\n",
        "emb_model = model.layers[4]\n",
        "\n",
        "# The original model requiers triplet input. In the inference model we want to pass only one single image\n",
        "# Model's output will be image's embedding vector\n",
        "# We now can compare the \"signature\" of the test image to the embeddings of the whole dataset and find the most similar one\n",
        "\n",
        "infer_model = Model(inp, emb_model(base_model(inp)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xLigUdL99qOE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "img = image.img_to_array(img).astype('uint8')\n",
        "x = np.expand_dims(img, axis=0)\n",
        "x = preprocess_input(x)\n",
        "\n",
        "pred = infer_model.predict(x)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pr4zF5It9yad",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 807
        },
        "outputId": "3620095b-44c7-4ef3-d598-6bdf881ed6c4"
      },
      "source": [
        "pred"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[-9.26755071e-02,  3.72152813e-02,  2.81159699e-01,\n",
              "        -4.66661938e-02,  2.59193093e-01, -2.20203772e-02,\n",
              "         4.67366315e-02, -9.38763097e-02,  1.46682560e-01,\n",
              "        -9.53742638e-02, -5.44019230e-02, -1.79618839e-02,\n",
              "        -1.29414827e-01,  6.51171431e-02,  2.18846258e-02,\n",
              "         1.14600584e-01,  6.63938224e-02, -8.55303630e-02,\n",
              "        -5.69820441e-02, -1.66619346e-02,  9.80565092e-04,\n",
              "         1.37820482e-01,  5.31403907e-02, -4.83523682e-02,\n",
              "         3.72724198e-02, -1.06248550e-01,  5.66537128e-05,\n",
              "        -1.48238122e-01,  1.54392928e-01,  1.12121925e-02,\n",
              "         2.67801285e-02,  3.62332532e-04, -7.36613572e-02,\n",
              "         1.26444027e-01, -8.62387642e-02,  8.54930803e-02,\n",
              "        -4.37724814e-02, -4.47887369e-02,  3.75721464e-03,\n",
              "        -7.91299194e-02, -6.32280558e-02,  8.99815261e-02,\n",
              "         2.16358923e-03, -1.26731992e-01, -5.98444641e-02,\n",
              "         1.22724950e-01,  4.17794101e-02,  4.25567180e-02,\n",
              "        -9.86644030e-02, -5.70189673e-03, -1.45639628e-01,\n",
              "        -1.52571173e-02,  9.69331041e-02, -6.21892395e-04,\n",
              "        -7.70248622e-02, -2.03765690e-01, -3.76918241e-02,\n",
              "         1.09458461e-01, -6.18636701e-03,  2.11225197e-01,\n",
              "        -1.49076909e-01, -4.42709140e-02, -5.47304153e-02,\n",
              "         1.47605399e-02,  3.56603116e-02, -3.10998000e-02,\n",
              "         1.02733821e-01, -5.47305448e-03, -1.72739252e-02,\n",
              "        -1.35036051e-01, -1.51015773e-01,  2.94131376e-02,\n",
              "        -1.01030171e-01,  1.81720451e-01,  1.13285612e-02,\n",
              "        -2.49479469e-02, -2.42109615e-02,  9.02575329e-02,\n",
              "         1.28055185e-01, -6.10964336e-02, -9.62763876e-02,\n",
              "        -1.03448212e-01,  5.38454577e-02,  2.02253480e-02,\n",
              "        -2.34625190e-02, -1.53563529e-01,  1.43879522e-02,\n",
              "        -4.25678939e-02,  8.70036110e-02, -3.49570662e-02,\n",
              "        -1.61149818e-02, -2.26054471e-02,  1.01831689e-01,\n",
              "         1.22219035e-02,  1.53310746e-01, -1.56182110e-01,\n",
              "         8.90679508e-02, -2.83490866e-02,  1.26597509e-01,\n",
              "        -4.53698896e-02, -9.67190862e-02, -1.38393305e-02,\n",
              "        -1.20121583e-01,  2.80799493e-02,  6.43855780e-02,\n",
              "        -9.46743414e-02, -2.94131017e-03, -3.67270447e-02,\n",
              "        -1.31531404e-02, -4.58859913e-02, -6.58286139e-02,\n",
              "        -7.03442201e-04, -2.76670475e-02, -2.39135586e-02,\n",
              "        -1.80588607e-02, -9.98960063e-03, -7.09511042e-02,\n",
              "         6.44643828e-02, -1.50644975e-02, -1.56779513e-01,\n",
              "        -1.58399101e-02,  1.46331847e-01,  8.88406858e-02,\n",
              "         1.38407317e-03, -6.33292459e-03,  8.12440813e-02,\n",
              "         5.26042730e-02, -1.46629158e-04]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tsmWAZkq_cx_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "img = tf.keras.preprocessing.image.load_img('{}'.format(values[1][0])).resize((178,218))\n",
        "img = image.img_to_array(img).astype('uint8')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MuHRy31m_tXv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y = np.expand_dims(img, axis=0)\n",
        "y = preprocess_input(y)\n",
        "\n",
        "pred_y = infer_model.predict(y)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WUZtVapu_0Tb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pred_y"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8lCAOIzI_1qX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.metrics.pairwise import cosine_similarity"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "edUmBEVqALU4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "66ad6b4a-d8ea-4725-f426-342f2ff236ab"
      },
      "source": [
        "cosine_similarity(pred,pred_y)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.73577464]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n5cm1fQJAUBR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "9ee88771-7f7f-4ec6-a530-4ebaefe93543"
      },
      "source": [
        "values[3]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['lfw/Gordon_Brown/Gordon_Brown_0002.jpg', 3]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mydfjhxmAXKw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "img = tf.keras.preprocessing.image.load_img('{}'.format(values[2][0])).resize((178,218))\n",
        "img = image.img_to_array(img).astype('uint8')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fim7D9xpA4aN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y = np.expand_dims(img, axis=0)\n",
        "y = preprocess_input(y)\n",
        "\n",
        "pred_y = infer_model.predict(y)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KRrwxPyVA7TG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "img = tf.keras.preprocessing.image.load_img('{}'.format(values[3][0])).resize((178,218))\n",
        "img = image.img_to_array(img).astype('uint8')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2D4N68qOA_Du",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bb83b297-8cf5-4e30-c0b4-67cb42f8cdc7"
      },
      "source": [
        "x = np.expand_dims(img, axis=0)\n",
        "x = preprocess_input(x)\n",
        "\n",
        "pred_y = infer_model.predict(x)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-1-021831b51b2f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpand_dims\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpreprocess_input\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mpred_y\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minfer_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'np' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l_c8bOwPBEm0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 237
        },
        "outputId": "2e8490b6-876e-43fe-f4b6-ab2dabf1a6ab"
      },
      "source": [
        "!pip install mtcnn"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting mtcnn\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/67/43/abee91792797c609c1bf30f1112117f7a87a713ebaa6ec5201d5555a73ef/mtcnn-0.1.0-py3-none-any.whl (2.3MB)\n",
            "\u001b[K     |████████████████████████████████| 2.3MB 3.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: keras>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from mtcnn) (2.4.3)\n",
            "Requirement already satisfied: opencv-python>=4.1.0 in /usr/local/lib/python3.6/dist-packages (from mtcnn) (4.1.2.30)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from keras>=2.0.0->mtcnn) (3.13)\n",
            "Requirement already satisfied: scipy>=0.14 in /usr/local/lib/python3.6/dist-packages (from keras>=2.0.0->mtcnn) (1.4.1)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras>=2.0.0->mtcnn) (2.10.0)\n",
            "Requirement already satisfied: numpy>=1.9.1 in /usr/local/lib/python3.6/dist-packages (from keras>=2.0.0->mtcnn) (1.18.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from h5py->keras>=2.0.0->mtcnn) (1.15.0)\n",
            "Installing collected packages: mtcnn\n",
            "Successfully installed mtcnn-0.1.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BMXKTMLjvV2f",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        " from mtcnn import MTCNN\n",
        " import cv2\n",
        "from google.colab.patches import cv2_imshow"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5XDQt1IGvv3c",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "detector = MTCNN()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZmSOSdDSvr7q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "img = cv2.cvtColor(cv2.imread(\"adam.jpg\"), cv2.COLOR_BGR2RGB)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EGL0SUANvxcs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "faces = detector.detect_faces(img)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kmGQwCXwwejS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "faces.sort(key=lambda x : x['confidence'] ,reverse=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S4vxAXlhzrE2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 274
        },
        "outputId": "8624c6e5-9e05-4e0d-c853-59deed896cb6"
      },
      "source": [
        "faces"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'box': [181, 1399, 219, 286],\n",
              "  'confidence': 0.999778687953949,\n",
              "  'keypoints': {'left_eye': (248, 1508),\n",
              "   'mouth_left': (261, 1619),\n",
              "   'mouth_right': (342, 1621),\n",
              "   'nose': (303, 1557),\n",
              "   'right_eye': (352, 1507)}},\n",
              " {'box': [1016, 411, 226, 293],\n",
              "  'confidence': 0.9034368395805359,\n",
              "  'keypoints': {'left_eye': (1080, 522),\n",
              "   'mouth_left': (1083, 638),\n",
              "   'mouth_right': (1165, 640),\n",
              "   'nose': (1128, 580),\n",
              "   'right_eye': (1184, 521)}}]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "slcjIp4Yv7E5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 267
        },
        "outputId": "eff72d5f-8138-4168-a9aa-83fd704a8e90"
      },
      "source": [
        "f = faces[0]\n",
        "x, y, width, height = f['box']\n",
        "roi = img[y:y+height, x:x+width]\n",
        "roi = cv2.cvtColor(roi, cv2.COLOR_BGR2RGB)\n",
        "roi_a = cv2.resize(roi, (250, 250)) \n",
        "cv2_imshow(roi_a)"
      ],
      "execution_count": null,
      "outputs": [
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BnIAr5dbwhvb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "org = cv2.cvtColor(cv2.imread(\"org.jpg\"), cv2.COLOR_BGR2RGB)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "95T-4m9vxDrr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "faces_org = detector.detect_faces(org)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XkcUPz9Bz4DL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "faces_org.sort(key=lambda x : x['confidence'] ,reverse=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RB9L8LttxyJL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 274
        },
        "outputId": "386afa64-62d1-47c4-d38b-5dd9b8d9d4ac"
      },
      "source": [
        "faces_org"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'box': [696, 952, 957, 1301],\n",
              "  'confidence': 0.9999947547912598,\n",
              "  'keypoints': {'left_eye': (938, 1480),\n",
              "   'mouth_left': (994, 1979),\n",
              "   'mouth_right': (1352, 1981),\n",
              "   'nose': (1169, 1704),\n",
              "   'right_eye': (1379, 1472)}},\n",
              " {'box': [49, -118, 619, 661],\n",
              "  'confidence': 0.8415143489837646,\n",
              "  'keypoints': {'left_eye': (453, 80),\n",
              "   'mouth_left': (401, 276),\n",
              "   'mouth_right': (503, 320),\n",
              "   'nose': (573, 172),\n",
              "   'right_eye': (558, 138)}}]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EW_BbSYTxHWe",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 267
        },
        "outputId": "59f8bb87-f25a-44a4-8ff4-c40afbaef275"
      },
      "source": [
        "f = faces_org[0]\n",
        "x, y, width, height = f['box']\n",
        "roi = org[y:y+height, x:x+width]\n",
        "roi = cv2.cvtColor(roi, cv2.COLOR_BGR2RGB)\n",
        "roi_b = cv2.resize(roi, (250, 250)) \n",
        "cv2_imshow(roi_b )"
      ],
      "execution_count": null,
      "outputs": [
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mqklbcz7xLel",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 74
        },
        "outputId": "07aa5e43-e746-4932-9df7-57fc8ffadb1a"
      },
      "source": [
        "img = image.img_to_array(roi_a).astype('uint8')\n",
        "x = np.expand_dims(img, axis=0)\n",
        "x = preprocess_input(x)\n",
        "\n",
        "pred = infer_model.predict(x)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Model was constructed with shape (None, 218, 178, 3) for input Tensor(\"anchor:0\", shape=(None, 218, 178, 3), dtype=float32), but it was called on an input with incompatible shape (None, 250, 250, 3).\n",
            "WARNING:tensorflow:Model was constructed with shape (None, 218, 178, 3) for input Tensor(\"input_7:0\", shape=(None, 218, 178, 3), dtype=float32), but it was called on an input with incompatible shape (None, 250, 250, 3).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8QZU3Jtm1K4t",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 495
        },
        "outputId": "20f830eb-6b10-4e96-cb81-4cbdf12fc440"
      },
      "source": [
        "pred"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[-0.09816048,  0.14464018,  0.1704588 ,  0.00962011,  0.13943703,\n",
              "        -0.09563027,  0.173818  , -0.04725586,  0.02563296, -0.107278  ,\n",
              "         0.05921918, -0.00723996, -0.08596681,  0.0531845 ,  0.05203275,\n",
              "         0.08431533,  0.05214313, -0.07612567,  0.02379467, -0.02555797,\n",
              "        -0.00348995,  0.09270043,  0.07262781,  0.07838209,  0.01791501,\n",
              "        -0.067817  ,  0.01207569,  0.01787226, -0.00347056, -0.04700113,\n",
              "         0.01070395,  0.01739817, -0.09225466,  0.02684948, -0.104043  ,\n",
              "         0.04301666, -0.05922959, -0.05549469,  0.17909586, -0.03660172,\n",
              "        -0.07468255,  0.09883451, -0.06258297, -0.07681803, -0.10267445,\n",
              "         0.18950889, -0.0905662 ,  0.04980819, -0.16484809,  0.02548244,\n",
              "        -0.08393566,  0.01689506,  0.1874739 , -0.04897448, -0.14680663,\n",
              "        -0.20626432,  0.01306755,  0.08543795,  0.05111904,  0.07685867,\n",
              "        -0.10522561, -0.16483036, -0.00093882, -0.01263139,  0.07587713,\n",
              "        -0.04351643,  0.01853183, -0.0327722 , -0.17932065, -0.14986983,\n",
              "        -0.10507258, -0.00267232,  0.00964544,  0.23640686, -0.0851677 ,\n",
              "        -0.06710515, -0.03322623,  0.14295739,  0.11071599, -0.01912762,\n",
              "         0.0095689 , -0.00404608,  0.17151138, -0.02098972,  0.0384587 ,\n",
              "        -0.18533012, -0.04915262, -0.02132475,  0.08731189, -0.05518897,\n",
              "         0.05523454,  0.01704978,  0.07068211,  0.02546095,  0.12669338,\n",
              "        -0.01746619,  0.14803316,  0.00473659, -0.00253292, -0.0631636 ,\n",
              "         0.00579346, -0.04416796, -0.11651712,  0.08400706,  0.09883281,\n",
              "        -0.02354331,  0.1076855 , -0.08229902,  0.06100487, -0.04245611,\n",
              "        -0.01745724, -0.07452319, -0.11321276, -0.01956922, -0.14204133,\n",
              "         0.04924611, -0.03151383,  0.12274852, -0.0271031 , -0.03347697,\n",
              "        -0.02954425,  0.20552209,  0.02269859, -0.02521807, -0.01824023,\n",
              "         0.0133437 , -0.03962522,  0.01801748]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R19JBNxl1MJP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "img = image.img_to_array(roi_b).astype('uint8')\n",
        "x = np.expand_dims(img, axis=0)\n",
        "x = preprocess_input(x)\n",
        "\n",
        "pred_b = infer_model.predict(x)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8G7rRDFU1Q4C",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "ef5f6b5d-20db-4d38-b644-6c4ce27ab94b"
      },
      "source": [
        "cosine_similarity(pred,pred_b)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.97465014]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E8zSECjT1ZIL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}