7_Facial Verification_code.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.1 Install Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: tensorflow in /home/dsm/.local/lib/python3.10/site-packages (2.10.0)\n",
      "Requirement already satisfied: matplotlib in /usr/lib/python3/dist-packages (3.5.1)\n",
      "Requirement already satisfied: tensorflow-estimator<2.11,>=2.10.0 in /home/dsm/.local/lib/python3.10/site-packages (from tensorflow) (2.10.0)\n",
      "Requirement already satisfied: gast<=0.4.0,>=0.2.1 in /home/dsm/.local/lib/python3.10/site-packages (from tensorflow) (0.4.0)\n",
      "Requirement already satisfied: h5py>=2.9.0 in /home/dsm/.local/lib/python3.10/site-packages (from tensorflow) (3.7.0)\n",
      "Requirement already satisfied: packaging in /home/dsm/.local/lib/python3.10/site-packages (from tensorflow) (21.3)\n",
      "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /home/dsm/.local/lib/python3.10/site-packages (from tensorflow) (1.50.0)\n",
      "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /home/dsm/.local/lib/python3.10/site-packages (from tensorflow) (0.27.0)\n",
      "Requirement already satisfied: absl-py>=1.0.0 in /home/dsm/.local/lib/python3.10/site-packages (from tensorflow) (1.3.0)\n",
      "Requirement already satisfied: tensorboard<2.11,>=2.10 in /home/dsm/.local/lib/python3.10/site-packages (from tensorflow) (2.10.1)\n",
      "Requirement already satisfied: flatbuffers>=2.0 in /home/dsm/.local/lib/python3.10/site-packages (from tensorflow) (22.10.26)\n",
      "Requirement already satisfied: keras-preprocessing>=1.1.1 in /home/dsm/.local/lib/python3.10/site-packages (from tensorflow) (1.1.2)\n",
      "Requirement already satisfied: six>=1.12.0 in /usr/lib/python3/dist-packages (from tensorflow) (1.16.0)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /home/dsm/.local/lib/python3.10/site-packages (from tensorflow) (2.1.0)\n",
      "Requirement already satisfied: setuptools in /usr/lib/python3/dist-packages (from tensorflow) (59.6.0)\n",
      "Requirement already satisfied: typing-extensions>=3.6.6 in /home/dsm/.local/lib/python3.10/site-packages (from tensorflow) (4.4.0)\n",
      "Requirement already satisfied: protobuf<3.20,>=3.9.2 in /usr/lib/python3/dist-packages (from tensorflow) (3.12.4)\n",
      "Requirement already satisfied: keras<2.11,>=2.10.0 in /home/dsm/.local/lib/python3.10/site-packages (from tensorflow) (2.10.0)\n",
      "Requirement already satisfied: wrapt>=1.11.0 in /home/dsm/.local/lib/python3.10/site-packages (from tensorflow) (1.14.1)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in /home/dsm/.local/lib/python3.10/site-packages (from tensorflow) (3.3.0)\n",
      "Requirement already satisfied: numpy>=1.20 in /usr/lib/python3/dist-packages (from tensorflow) (1.21.5)\n",
      "Requirement already satisfied: astunparse>=1.6.0 in /home/dsm/.local/lib/python3.10/site-packages (from tensorflow) (1.6.3)\n",
      "Requirement already satisfied: libclang>=13.0.0 in /home/dsm/.local/lib/python3.10/site-packages (from tensorflow) (14.0.6)\n",
      "Requirement already satisfied: google-pasta>=0.1.1 in /home/dsm/.local/lib/python3.10/site-packages (from tensorflow) (0.2.0)\n",
      "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/lib/python3/dist-packages (from astunparse>=1.6.0->tensorflow) (0.37.1)\n",
      "Requirement already satisfied: google-auth<3,>=1.6.3 in /home/dsm/.local/lib/python3.10/site-packages (from tensorboard<2.11,>=2.10->tensorflow) (2.14.0)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in /usr/lib/python3/dist-packages (from tensorboard<2.11,>=2.10->tensorflow) (2.25.1)\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in /home/dsm/.local/lib/python3.10/site-packages (from tensorboard<2.11,>=2.10->tensorflow) (2.2.2)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /home/dsm/.local/lib/python3.10/site-packages (from tensorboard<2.11,>=2.10->tensorflow) (3.4.1)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /home/dsm/.local/lib/python3.10/site-packages (from tensorboard<2.11,>=2.10->tensorflow) (0.4.6)\n",
      "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /home/dsm/.local/lib/python3.10/site-packages (from tensorboard<2.11,>=2.10->tensorflow) (0.6.1)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /home/dsm/.local/lib/python3.10/site-packages (from tensorboard<2.11,>=2.10->tensorflow) (1.8.1)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/lib/python3/dist-packages (from packaging->tensorflow) (2.4.7)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /home/dsm/.local/lib/python3.10/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.11,>=2.10->tensorflow) (4.9)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /home/dsm/.local/lib/python3.10/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.11,>=2.10->tensorflow) (5.2.0)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /home/dsm/.local/lib/python3.10/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.11,>=2.10->tensorflow) (0.2.8)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /home/dsm/.local/lib/python3.10/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.11,>=2.10->tensorflow) (1.3.1)\n",
      "Requirement already satisfied: MarkupSafe>=2.1.1 in /home/dsm/.local/lib/python3.10/site-packages (from werkzeug>=1.0.1->tensorboard<2.11,>=2.10->tensorflow) (2.1.1)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /home/dsm/.local/lib/python3.10/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.11,>=2.10->tensorflow) (0.4.8)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /usr/lib/python3/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.11,>=2.10->tensorflow) (3.2.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install tensorflow matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: opencv-python==4.5.5.64 in /home/dsm/.local/lib/python3.10/site-packages (4.5.5.64)\n",
      "Requirement already satisfied: numpy>=1.17.3 in /usr/lib/python3/dist-packages (from opencv-python==4.5.5.64) (1.21.5)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install opencv-python==4.5.5.64"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.2 Import Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import standard dependencies\n",
    "import cv2\n",
    "import os\n",
    "import random\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-11-05 05:13:19.108689: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /home/dsm/.local/lib/python3.10/site-packages/cv2/../../lib64:\n",
      "2022-11-05 05:13:19.108806: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n",
      "2022-11-05 05:13:19.300951: E tensorflow/stream_executor/cuda/cuda_blas.cc:2981] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2022-11-05 05:13:27.717980: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /home/dsm/.local/lib/python3.10/site-packages/cv2/../../lib64:\n",
      "2022-11-05 05:13:27.719355: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /home/dsm/.local/lib/python3.10/site-packages/cv2/../../lib64:\n",
      "2022-11-05 05:13:27.719455: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n"
     ]
    }
   ],
   "source": [
    "# Import tensorflow dependencies - Functional API\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.layers import Layer, Conv2D, Dense, MaxPooling2D, Input, Flatten\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.4 Create Folder Structures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setup paths\n",
    "POS_PATH = os.path.join('data', 'positive')\n",
    "NEG_PATH = os.path.join('data', 'negative')\n",
    "ANC_PATH = os.path.join('data', 'anchor')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileExistsError",
     "evalue": "[Errno 17] File exists: 'data/positive'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileExistsError\u001b[0m                           Traceback (most recent call last)",
      "Cell \u001b[0;32mIn [6], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# Make the directories\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m \u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmakedirs\u001b[49m\u001b[43m(\u001b[49m\u001b[43mPOS_PATH\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m      3\u001b[0m os\u001b[38;5;241m.\u001b[39mmakedirs(NEG_PATH)\n\u001b[1;32m      4\u001b[0m os\u001b[38;5;241m.\u001b[39mmakedirs(ANC_PATH)\n",
      "File \u001b[0;32m/usr/lib/python3.10/os.py:225\u001b[0m, in \u001b[0;36mmakedirs\u001b[0;34m(name, mode, exist_ok)\u001b[0m\n\u001b[1;32m    223\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m\n\u001b[1;32m    224\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 225\u001b[0m     \u001b[43mmkdir\u001b[49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    226\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m:\n\u001b[1;32m    227\u001b[0m     \u001b[38;5;66;03m# Cannot rely on checking for EEXIST, since the operating system\u001b[39;00m\n\u001b[1;32m    228\u001b[0m     \u001b[38;5;66;03m# could give priority to other errors like EACCES or EROFS\u001b[39;00m\n\u001b[1;32m    229\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m exist_ok \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m path\u001b[38;5;241m.\u001b[39misdir(name):\n",
      "\u001b[0;31mFileExistsError\u001b[0m: [Errno 17] File exists: 'data/positive'"
     ]
    }
   ],
   "source": [
    "# Make the directories\n",
    "os.makedirs(POS_PATH)\n",
    "os.makedirs(NEG_PATH)\n",
    "os.makedirs(ANC_PATH)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Collect Positives and Anchors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.1 Untar Labelled Faces in the Wild Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# http://vis-www.cs.umass.edu/lfw/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Uncompress Tar GZ Labelled Faces in the Wild Dataset\n",
    "!tar -xf lfw.tgz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Move LFW Images to the following repository data/negative\n",
    "for directory in os.listdir('lfw'):\n",
    "    for file in os.listdir(os.path.join('lfw', directory)):\n",
    "        EX_PATH = os.path.join('lfw', directory, file)\n",
    "        NEW_PATH = os.path.join(NEG_PATH, file)\n",
    "        os.replace(EX_PATH, NEW_PATH)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2 Collect Positive and Anchor Classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import uuid library to generate unique image names\n",
    "import uuid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'data/anchor/c98adc60-5caf-11ed-bed8-0103e5437baf.jpg'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.path.join(ANC_PATH, '{}.jpg'.format(uuid.uuid1()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: opencv-contrib-python in /home/dsm/.local/lib/python3.10/site-packages (4.6.0.66)\n",
      "Requirement already satisfied: numpy>=1.14.5 in /usr/lib/python3/dist-packages (from opencv-contrib-python) (1.21.5)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install opencv-contrib-python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: Ignoring XDG_SESSION_TYPE=wayland on Gnome. Use QT_QPA_PLATFORM=wayland to run on Wayland anyway.\n"
     ]
    }
   ],
   "source": [
    "# Establish a connection to the webcam\n",
    "cap = cv2.VideoCapture(-1)\n",
    "while cap.isOpened(): \n",
    "    ret, frame = cap.read()\n",
    "   \n",
    "    # Cut down frame to 250x250px\n",
    "    frame = frame[120:120+250,200:200+250, :]\n",
    "    \n",
    "    # Collect anchors \n",
    "    if cv2.waitKey(1) & 0XFF == ord('a'):\n",
    "        # Create the unique file path \n",
    "        imgname = os.path.join(ANC_PATH, '{}.jpg'.format(uuid.uuid1()))\n",
    "        # Write out anchor image\n",
    "        cv2.imwrite(imgname, frame)\n",
    "    \n",
    "    # Collect positives\n",
    "    if cv2.waitKey(1) & 0XFF == ord('p'):\n",
    "        # Create the unique file path \n",
    "        imgname = os.path.join(POS_PATH, '{}.jpg'.format(uuid.uuid1()))\n",
    "        # Write out positive image\n",
    "        cv2.imwrite(imgname, frame)\n",
    "    \n",
    "    # Show image back to screen\n",
    "    cv2.imshow('Image Collection', frame)\n",
    "    \n",
    "    # Breaking gracefully\n",
    "    if cv2.waitKey(1) & 0XFF == ord('q'):\n",
    "        break\n",
    "        \n",
    "# Release the webcam\n",
    "cap.release()\n",
    "# Close the image show frame\n",
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(frame[120:120+250,200:200+250, :])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Load and Preprocess Images"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.1 Get Image Directories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "anchor = tf.data.Dataset.list_files(ANC_PATH+'\\*.jpg').take(300)\n",
    "positive = tf.data.Dataset.list_files(POS_PATH+'\\*.jpg').take(300)\n",
    "negative = tf.data.Dataset.list_files(NEG_PATH+'\\*.jpg').take(300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_test = anchor.as_numpy_iterator()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(dir_test.next())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.2 Preprocessing - Scale and Resize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(file_path):\n",
    "    \n",
    "    # Read in image from file path\n",
    "    byte_img = tf.io.read_file(file_path)\n",
    "    # Load in the image \n",
    "    img = tf.io.decode_jpeg(byte_img)\n",
    "    \n",
    "    # Preprocessing steps - resizing the image to be 100x100x3\n",
    "    img = tf.image.resize(img, (100,100))\n",
    "    # Scale image to be between 0 and 1 \n",
    "    img = img / 255.0\n",
    "    \n",
    "    # Return image\n",
    "    return img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "img = preprocess('data\\\\anchor\\\\a4e73462-135f-11ec-9e6e-a0cec8d2d278.jpg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "img.numpy().max() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.map(preprocess)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.3 Create Labelled Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# (anchor, positive) => 1,1,1,1,1\n",
    "# (anchor, negative) => 0,0,0,0,0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "positives = tf.data.Dataset.zip((anchor, positive, tf.data.Dataset.from_tensor_slices(tf.ones(len(anchor)))))\n",
    "negatives = tf.data.Dataset.zip((anchor, negative, tf.data.Dataset.from_tensor_slices(tf.zeros(len(anchor)))))\n",
    "data = positives.concatenate(negatives)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "samples = data.as_numpy_iterator()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "exampple = samples.next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "exampple"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.4 Build Train and Test Partition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_twin(input_img, validation_img, label):\n",
    "    return(preprocess(input_img), preprocess(validation_img), label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = preprocess_twin(*exampple)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(res[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build dataloader pipeline\n",
    "data = data.map(preprocess_twin)\n",
    "data = data.cache()\n",
    "data = data.shuffle(buffer_size=1024)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training partition\n",
    "train_data = data.take(round(len(data)*.7))\n",
    "train_data = train_data.batch(16)\n",
    "train_data = train_data.prefetch(8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Testing partition\n",
    "test_data = data.skip(round(len(data)*.7))\n",
    "test_data = test_data.take(round(len(data)*.3))\n",
    "test_data = test_data.batch(16)\n",
    "test_data = test_data.prefetch(8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Model Engineering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.1 Build Embedding Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inp = Input(shape=(100,100,3), name='input_image')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "c1 = Conv2D(64, (10,10), activation='relu')(inp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m1 = MaxPooling2D(64, (2,2), padding='same')(c1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "c2 = Conv2D(128, (7,7), activation='relu')(m1)\n",
    "m2 = MaxPooling2D(64, (2,2), padding='same')(c2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "c3 = Conv2D(128, (4,4), activation='relu')(m2)\n",
    "m3 = MaxPooling2D(64, (2,2), padding='same')(c3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "c4 = Conv2D(256, (4,4), activation='relu')(m3)\n",
    "f1 = Flatten()(c4)\n",
    "d1 = Dense(4096, activation='sigmoid')(f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mod = Model(inputs=[inp], outputs=[d1], name='embedding')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mod.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_embedding(): \n",
    "    inp = Input(shape=(100,100,3), name='input_image')\n",
    "    \n",
    "    # First block\n",
    "    c1 = Conv2D(64, (10,10), activation='relu')(inp)\n",
    "    m1 = MaxPooling2D(64, (2,2), padding='same')(c1)\n",
    "    \n",
    "    # Second block\n",
    "    c2 = Conv2D(128, (7,7), activation='relu')(m1)\n",
    "    m2 = MaxPooling2D(64, (2,2), padding='same')(c2)\n",
    "    \n",
    "    # Third block \n",
    "    c3 = Conv2D(128, (4,4), activation='relu')(m2)\n",
    "    m3 = MaxPooling2D(64, (2,2), padding='same')(c3)\n",
    "    \n",
    "    # Final embedding block\n",
    "    c4 = Conv2D(256, (4,4), activation='relu')(m3)\n",
    "    f1 = Flatten()(c4)\n",
    "    d1 = Dense(4096, activation='sigmoid')(f1)\n",
    "    \n",
    "    \n",
    "    return Model(inputs=[inp], outputs=[d1], name='embedding')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding = make_embedding()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.2 Build Distance Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Siamese L1 Distance class\n",
    "class L1Dist(Layer):\n",
    "    \n",
    "    # Init method - inheritance\n",
    "    def __init__(self, **kwargs):\n",
    "        super().__init__()\n",
    "       \n",
    "    # Magic happens here - similarity calculation\n",
    "    def call(self, input_embedding, validation_embedding):\n",
    "        return tf.math.abs(input_embedding - validation_embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "l1 = L1Dist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "l1(anchor_embedding, validation_embedding)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.3 Make Siamese Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_image = Input(name='input_img', shape=(100,100,3))\n",
    "validation_image = Input(name='validation_img', shape=(100,100,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inp_embedding = embedding(input_image)\n",
    "val_embedding = embedding(validation_image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "siamese_layer = L1Dist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "distances = siamese_layer(inp_embedding, val_embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier = Dense(1, activation='sigmoid')(distances)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "siamese_network = Model(inputs=[input_image, validation_image], outputs=classifier, name='SiameseNetwork')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "siamese_network.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_siamese_model(): \n",
    "    \n",
    "    # Anchor image input in the network\n",
    "    input_image = Input(name='input_img', shape=(100,100,3))\n",
    "    \n",
    "    # Validation image in the network \n",
    "    validation_image = Input(name='validation_img', shape=(100,100,3))\n",
    "    \n",
    "    # Combine siamese distance components\n",
    "    siamese_layer = L1Dist()\n",
    "    siamese_layer._name = 'distance'\n",
    "    distances = siamese_layer(embedding(input_image), embedding(validation_image))\n",
    "    \n",
    "    # Classification layer \n",
    "    classifier = Dense(1, activation='sigmoid')(distances)\n",
    "    \n",
    "    return Model(inputs=[input_image, validation_image], outputs=classifier, name='SiameseNetwork')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "siamese_model = make_siamese_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "siamese_model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.1 Setup Loss and Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "binary_cross_loss = tf.losses.BinaryCrossentropy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt = tf.keras.optimizers.Adam(1e-4) # 0.0001"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.2 Establish Checkpoints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint_dir = './training_checkpoints'\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, 'ckpt')\n",
    "checkpoint = tf.train.Checkpoint(opt=opt, siamese_model=siamese_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.3 Build Train Step Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_batch = train_data.as_numpy_iterator()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_1 = test_batch.next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = batch_1[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = batch_1[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.losses.BinaryCrossentropy??"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def train_step(batch):\n",
    "    \n",
    "    # Record all of our operations \n",
    "    with tf.GradientTape() as tape:     \n",
    "        # Get anchor and positive/negative image\n",
    "        X = batch[:2]\n",
    "        # Get label\n",
    "        y = batch[2]\n",
    "        \n",
    "        # Forward pass\n",
    "        yhat = siamese_model(X, training=True)\n",
    "        # Calculate loss\n",
    "        loss = binary_cross_loss(y, yhat)\n",
    "    print(loss)\n",
    "        \n",
    "    # Calculate gradients\n",
    "    grad = tape.gradient(loss, siamese_model.trainable_variables)\n",
    "    \n",
    "    # Calculate updated weights and apply to siamese model\n",
    "    opt.apply_gradients(zip(grad, siamese_model.trainable_variables))\n",
    "    \n",
    "    # Return loss\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.4 Build Training Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(data, EPOCHS):\n",
    "    # Loop through epochs\n",
    "    for epoch in range(1, EPOCHS+1):\n",
    "        print('\\n Epoch {}/{}'.format(epoch, EPOCHS))\n",
    "        progbar = tf.keras.utils.Progbar(len(data))\n",
    "        \n",
    "        # Loop through each batch\n",
    "        for idx, batch in enumerate(data):\n",
    "            # Run train step here\n",
    "            train_step(batch)\n",
    "            progbar.update(idx+1)\n",
    "        \n",
    "        # Save checkpoints\n",
    "        if epoch % 10 == 0: \n",
    "            checkpoint.save(file_prefix=checkpoint_prefix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.5 Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "EPOCHS = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train(train_data, EPOCHS)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6. Evaluate Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6.1 Import Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import metric calculations\n",
    "from tensorflow.keras.metrics import Precision, Recall"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6.2 Make Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get a batch of test data\n",
    "test_input, test_val, y_true = test_data.as_numpy_iterator().next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make predictions\n",
    "y_hat = siamese_model.predict([test_input, test_val])\n",
    "y_hat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Post processing the results \n",
    "[1 if prediction > 0.5 else 0 for prediction in y_hat ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6.3 Calculate Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a metric object \n",
    "m = Recall()\n",
    "\n",
    "# Calculating the recall value \n",
    "m.update_state(y_true, y_hat)\n",
    "\n",
    "# Return Recall Result\n",
    "m.result().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a metric object \n",
    "m = Precision()\n",
    "\n",
    "# Calculating the recall value \n",
    "m.update_state(y_true, y_hat)\n",
    "\n",
    "# Return Recall Result\n",
    "m.result().numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6.4 Viz Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set plot size \n",
    "plt.figure(figsize=(10,8))\n",
    "\n",
    "# Set first subplot\n",
    "plt.subplot(1,2,1)\n",
    "plt.imshow(test_input[0])\n",
    "\n",
    "# Set second subplot\n",
    "plt.subplot(1,2,2)\n",
    "plt.imshow(test_val[0])\n",
    "\n",
    "# Renders cleanly\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 7. Save Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save weights\n",
    "siamese_model.save('siamesemodel.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "L1Dist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reload model \n",
    "model = tf.keras.models.load_model('siamesemodel.h5', \n",
    "                                   custom_objects={'L1Dist':L1Dist, 'BinaryCrossentropy':tf.losses.BinaryCrossentropy})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make predictions with reloaded model\n",
    "model.predict([test_input, test_val])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# View model summary\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 8. Real Time Test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 8.1 Verification Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_data\\verification_images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.listdir(os.path.join('application_data', 'verification_images'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.path.join('application_data', 'input_image', 'input_image.jpg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for image in os.listdir(os.path.join('application_data', 'verification_images')):\n",
    "    validation_img = os.path.join('application_data', 'verification_images', image)\n",
    "    print(validation_img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def verify(model, detection_threshold, verification_threshold):\n",
    "    # Build results array\n",
    "    results = []\n",
    "    for image in os.listdir(os.path.join('application_data', 'verification_images')):\n",
    "        input_img = preprocess(os.path.join('application_data', 'input_image', 'input_image.jpg'))\n",
    "        validation_img = preprocess(os.path.join('application_data', 'verification_images', image))\n",
    "        \n",
    "        # Make Predictions \n",
    "        result = model.predict(list(np.expand_dims([input_img, validation_img], axis=1)))\n",
    "        results.append(result)\n",
    "    \n",
    "    # Detection Threshold: Metric above which a prediciton is considered positive \n",
    "    detection = np.sum(np.array(results) > detection_threshold)\n",
    "    \n",
    "    # Verification Threshold: Proportion of positive predictions / total positive samples \n",
    "    verification = detection / len(os.listdir(os.path.join('application_data', 'verification_images'))) \n",
    "    verified = verification > verification_threshold\n",
    "    \n",
    "    return results, verified"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 8.2 OpenCV Real Time Verification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cap = cv2.VideoCapture(4)\n",
    "while cap.isOpened():\n",
    "    ret, frame = cap.read()\n",
    "    frame = frame[120:120+250,200:200+250, :]\n",
    "    \n",
    "    cv2.imshow('Verification', frame)\n",
    "    \n",
    "    # Verification trigger\n",
    "    if cv2.waitKey(10) & 0xFF == ord('v'):\n",
    "        # Save input image to application_data/input_image folder \n",
    "        cv2.imwrite(os.path.join('application_data', 'input_image', 'input_image.jpg'), frame)\n",
    "        # Run verification\n",
    "        results, verified = verify(model, 0.9, 0.7)\n",
    "        print(verified)\n",
    "    \n",
    "    if cv2.waitKey(10) & 0xFF == ord('q'):\n",
    "        break\n",
    "cap.release()\n",
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.sum(np.squeeze(results) > 0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "32/50"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}