transfer_iiit_pet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "\n",
    "# append PYTHONPATH to load extensions\n",
    "sys.path.append('fastaugment')\n",
    "sys.path.append('sigmoid_like_tf_op')\n",
    "\n",
    "# load TensorFlow\n",
    "import tensorflow as tf\n",
    "physical_devices = tf.config.list_physical_devices('GPU')\n",
    "tf.config.experimental.set_memory_growth(physical_devices[0], True)\n",
    "\n",
    "# import other modules\n",
    "import numpy\n",
    "import tensorflow_datasets as tfds\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow.data.experimental import AUTOTUNE\n",
    "import model\n",
    "from fast_augment import augment, center_crop\n",
    "from sigmoid_like import sigmoid_like"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"functional_3\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input (InputLayer)              [(None, 385, 385, 3) 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "b0-conv (Conv2D)                (None, 191, 191, 32) 2400        input[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "b0-conv-bn (BatchNormalization) (None, 191, 191, 32) 128         b0-conv[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "b0-conv-act (Activation)        (None, 191, 191, 32) 0           b0-conv-bn[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "b1-stage1-pw (Conv2D)           (None, 191, 191, 32) 1024        b0-conv-act[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "b1-stage1-pw-bn (BatchNormaliza (None, 191, 191, 32) 128         b1-stage1-pw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b1-stage1-pw-act (Activation)   (None, 191, 191, 32) 0           b1-stage1-pw-bn[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "b1-stage1-dw (Conv2D)           (None, 191, 191, 32) 1152        b1-stage1-pw-act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "b1-stage1-dw-bn (BatchNormaliza (None, 191, 191, 32) 128         b1-stage1-dw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b1-stage1-add (Add)             (None, 191, 191, 32) 0           b1-stage1-dw-bn[0][0]            \n",
      "                                                                 b0-conv-act[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "b1-stage1-act (Activation)      (None, 191, 191, 32) 0           b1-stage1-add[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b1-stage2-pw (Conv2D)           (None, 191, 191, 32) 1024        b1-stage1-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b1-stage2-pw-bn (BatchNormaliza (None, 191, 191, 32) 128         b1-stage2-pw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b1-stage2-pw-act (Activation)   (None, 191, 191, 32) 0           b1-stage2-pw-bn[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "b1-stage2-dw (Conv2D)           (None, 191, 191, 32) 1152        b1-stage2-pw-act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "b1-stage2-dw-bn (BatchNormaliza (None, 191, 191, 32) 128         b1-stage2-dw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b1-stage2-add (Add)             (None, 191, 191, 32) 0           b1-stage2-dw-bn[0][0]            \n",
      "                                                                 b1-stage1-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b1-stage2-act (Activation)      (None, 191, 191, 32) 0           b1-stage2-add[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b1-scale (Conv2D)               (None, 95, 95, 32)   2304        b1-stage2-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b1-scale-bn (BatchNormalization (None, 95, 95, 32)   128         b1-scale[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "b1-scale-act (Activation)       (None, 95, 95, 32)   0           b1-scale-bn[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "shuffle (Shuffle)               (None, 95, 95, 32)   0           b1-scale-act[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b2-stage1-pw (Conv2D)           (None, 95, 95, 32)   1024        shuffle[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "b2-stage1-pw-bn (BatchNormaliza (None, 95, 95, 32)   128         b2-stage1-pw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b2-stage1-pw-act (Activation)   (None, 95, 95, 32)   0           b2-stage1-pw-bn[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "b2-stage1-dw (Conv2D)           (None, 95, 95, 32)   1152        b2-stage1-pw-act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "b2-stage1-dw-bn (BatchNormaliza (None, 95, 95, 32)   128         b2-stage1-dw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b2-stage1-add (Add)             (None, 95, 95, 32)   0           b2-stage1-dw-bn[0][0]            \n",
      "                                                                 b1-scale-act[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b2-stage1-act (Activation)      (None, 95, 95, 32)   0           b2-stage1-add[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "shuffle_1 (Shuffle)             (None, 95, 95, 32)   0           b2-stage1-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b2-stage2-pw (Conv2D)           (None, 95, 95, 32)   1024        shuffle_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "b2-stage2-pw-bn (BatchNormaliza (None, 95, 95, 32)   128         b2-stage2-pw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b2-stage2-pw-act (Activation)   (None, 95, 95, 32)   0           b2-stage2-pw-bn[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "b2-stage2-dw (Conv2D)           (None, 95, 95, 32)   1152        b2-stage2-pw-act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "b2-stage2-dw-bn (BatchNormaliza (None, 95, 95, 32)   128         b2-stage2-dw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b2-stage2-add (Add)             (None, 95, 95, 32)   0           b2-stage2-dw-bn[0][0]            \n",
      "                                                                 b2-stage1-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b2-stage2-act (Activation)      (None, 95, 95, 32)   0           b2-stage2-add[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "shuffle_2 (Shuffle)             (None, 95, 95, 32)   0           b2-stage2-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b2-stage3-pw (Conv2D)           (None, 95, 95, 32)   1024        shuffle_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "b2-stage3-pw-bn (BatchNormaliza (None, 95, 95, 32)   128         b2-stage3-pw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b2-stage3-pw-act (Activation)   (None, 95, 95, 32)   0           b2-stage3-pw-bn[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "b2-stage3-dw (Conv2D)           (None, 95, 95, 32)   1152        b2-stage3-pw-act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "b2-stage3-dw-bn (BatchNormaliza (None, 95, 95, 32)   128         b2-stage3-dw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b2-stage3-add (Add)             (None, 95, 95, 32)   0           b2-stage3-dw-bn[0][0]            \n",
      "                                                                 b2-stage2-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b2-stage3-act (Activation)      (None, 95, 95, 32)   0           b2-stage3-add[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b2-scale (Conv2D)               (None, 47, 47, 64)   4608        b2-stage3-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b2-scale-bn (BatchNormalization (None, 47, 47, 64)   256         b2-scale[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "b2-scale-act (Activation)       (None, 47, 47, 64)   0           b2-scale-bn[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "shuffle_3 (Shuffle)             (None, 47, 47, 64)   0           b2-scale-act[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b3-stage1-pw (Conv2D)           (None, 47, 47, 64)   2048        shuffle_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "b3-stage1-pw-bn (BatchNormaliza (None, 47, 47, 64)   256         b3-stage1-pw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b3-stage1-pw-act (Activation)   (None, 47, 47, 64)   0           b3-stage1-pw-bn[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "b3-stage1-dw (Conv2D)           (None, 47, 47, 64)   2304        b3-stage1-pw-act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "b3-stage1-dw-bn (BatchNormaliza (None, 47, 47, 64)   256         b3-stage1-dw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b3-stage1-add (Add)             (None, 47, 47, 64)   0           b3-stage1-dw-bn[0][0]            \n",
      "                                                                 b2-scale-act[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b3-stage1-act (Activation)      (None, 47, 47, 64)   0           b3-stage1-add[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "shuffle_4 (Shuffle)             (None, 47, 47, 64)   0           b3-stage1-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b3-stage2-pw (Conv2D)           (None, 47, 47, 64)   2048        shuffle_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "b3-stage2-pw-bn (BatchNormaliza (None, 47, 47, 64)   256         b3-stage2-pw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b3-stage2-pw-act (Activation)   (None, 47, 47, 64)   0           b3-stage2-pw-bn[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "b3-stage2-dw (Conv2D)           (None, 47, 47, 64)   2304        b3-stage2-pw-act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "b3-stage2-dw-bn (BatchNormaliza (None, 47, 47, 64)   256         b3-stage2-dw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b3-stage2-add (Add)             (None, 47, 47, 64)   0           b3-stage2-dw-bn[0][0]            \n",
      "                                                                 b3-stage1-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b3-stage2-act (Activation)      (None, 47, 47, 64)   0           b3-stage2-add[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "shuffle_5 (Shuffle)             (None, 47, 47, 64)   0           b3-stage2-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b3-stage3-pw (Conv2D)           (None, 47, 47, 64)   2048        shuffle_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "b3-stage3-pw-bn (BatchNormaliza (None, 47, 47, 64)   256         b3-stage3-pw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b3-stage3-pw-act (Activation)   (None, 47, 47, 64)   0           b3-stage3-pw-bn[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "b3-stage3-dw (Conv2D)           (None, 47, 47, 64)   2304        b3-stage3-pw-act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "b3-stage3-dw-bn (BatchNormaliza (None, 47, 47, 64)   256         b3-stage3-dw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b3-stage3-add (Add)             (None, 47, 47, 64)   0           b3-stage3-dw-bn[0][0]            \n",
      "                                                                 b3-stage2-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b3-stage3-act (Activation)      (None, 47, 47, 64)   0           b3-stage3-add[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b3-scale (Conv2D)               (None, 23, 23, 128)  9216        b3-stage3-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b3-scale-bn (BatchNormalization (None, 23, 23, 128)  512         b3-scale[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "b3-scale-act (Activation)       (None, 23, 23, 128)  0           b3-scale-bn[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "shuffle_6 (Shuffle)             (None, 23, 23, 128)  0           b3-scale-act[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage1-pw (Conv2D)           (None, 23, 23, 128)  4096        shuffle_6[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage1-pw-bn (BatchNormaliza (None, 23, 23, 128)  512         b4-stage1-pw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage1-pw-act (Activation)   (None, 23, 23, 128)  0           b4-stage1-pw-bn[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage1-dw (Conv2D)           (None, 23, 23, 128)  4608        b4-stage1-pw-act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage1-dw-bn (BatchNormaliza (None, 23, 23, 128)  512         b4-stage1-dw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage1-add (Add)             (None, 23, 23, 128)  0           b4-stage1-dw-bn[0][0]            \n",
      "                                                                 b3-scale-act[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage1-act (Activation)      (None, 23, 23, 128)  0           b4-stage1-add[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "shuffle_7 (Shuffle)             (None, 23, 23, 128)  0           b4-stage1-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage2-pw (Conv2D)           (None, 23, 23, 128)  4096        shuffle_7[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage2-pw-bn (BatchNormaliza (None, 23, 23, 128)  512         b4-stage2-pw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage2-pw-act (Activation)   (None, 23, 23, 128)  0           b4-stage2-pw-bn[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage2-dw (Conv2D)           (None, 23, 23, 128)  4608        b4-stage2-pw-act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage2-dw-bn (BatchNormaliza (None, 23, 23, 128)  512         b4-stage2-dw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage2-add (Add)             (None, 23, 23, 128)  0           b4-stage2-dw-bn[0][0]            \n",
      "                                                                 b4-stage1-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage2-act (Activation)      (None, 23, 23, 128)  0           b4-stage2-add[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "shuffle_8 (Shuffle)             (None, 23, 23, 128)  0           b4-stage2-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage3-pw (Conv2D)           (None, 23, 23, 128)  4096        shuffle_8[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage3-pw-bn (BatchNormaliza (None, 23, 23, 128)  512         b4-stage3-pw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage3-pw-act (Activation)   (None, 23, 23, 128)  0           b4-stage3-pw-bn[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage3-dw (Conv2D)           (None, 23, 23, 128)  4608        b4-stage3-pw-act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage3-dw-bn (BatchNormaliza (None, 23, 23, 128)  512         b4-stage3-dw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage3-add (Add)             (None, 23, 23, 128)  0           b4-stage3-dw-bn[0][0]            \n",
      "                                                                 b4-stage2-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage3-act (Activation)      (None, 23, 23, 128)  0           b4-stage3-add[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "shuffle_9 (Shuffle)             (None, 23, 23, 128)  0           b4-stage3-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage4-pw (Conv2D)           (None, 23, 23, 128)  4096        shuffle_9[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage4-pw-bn (BatchNormaliza (None, 23, 23, 128)  512         b4-stage4-pw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage4-pw-act (Activation)   (None, 23, 23, 128)  0           b4-stage4-pw-bn[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage4-dw (Conv2D)           (None, 23, 23, 128)  4608        b4-stage4-pw-act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage4-dw-bn (BatchNormaliza (None, 23, 23, 128)  512         b4-stage4-dw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage4-add (Add)             (None, 23, 23, 128)  0           b4-stage4-dw-bn[0][0]            \n",
      "                                                                 b4-stage3-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b4-stage4-act (Activation)      (None, 23, 23, 128)  0           b4-stage4-add[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b4-scale (Conv2D)               (None, 11, 11, 192)  13824       b4-stage4-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b4-scale-bn (BatchNormalization (None, 11, 11, 192)  768         b4-scale[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "b4-scale-act (Activation)       (None, 11, 11, 192)  0           b4-scale-bn[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "shuffle_10 (Shuffle)            (None, 11, 11, 192)  0           b4-scale-act[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage1-pw (Conv2D)           (None, 11, 11, 192)  6144        shuffle_10[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage1-pw-bn (BatchNormaliza (None, 11, 11, 192)  768         b5-stage1-pw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage1-pw-act (Activation)   (None, 11, 11, 192)  0           b5-stage1-pw-bn[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage1-dw (Conv2D)           (None, 11, 11, 192)  6912        b5-stage1-pw-act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage1-dw-bn (BatchNormaliza (None, 11, 11, 192)  768         b5-stage1-dw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage1-add (Add)             (None, 11, 11, 192)  0           b5-stage1-dw-bn[0][0]            \n",
      "                                                                 b4-scale-act[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage1-act (Activation)      (None, 11, 11, 192)  0           b5-stage1-add[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "shuffle_11 (Shuffle)            (None, 11, 11, 192)  0           b5-stage1-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage2-pw (Conv2D)           (None, 11, 11, 192)  6144        shuffle_11[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage2-pw-bn (BatchNormaliza (None, 11, 11, 192)  768         b5-stage2-pw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage2-pw-act (Activation)   (None, 11, 11, 192)  0           b5-stage2-pw-bn[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage2-dw (Conv2D)           (None, 11, 11, 192)  6912        b5-stage2-pw-act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage2-dw-bn (BatchNormaliza (None, 11, 11, 192)  768         b5-stage2-dw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage2-add (Add)             (None, 11, 11, 192)  0           b5-stage2-dw-bn[0][0]            \n",
      "                                                                 b5-stage1-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage2-act (Activation)      (None, 11, 11, 192)  0           b5-stage2-add[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "shuffle_12 (Shuffle)            (None, 11, 11, 192)  0           b5-stage2-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage3-pw (Conv2D)           (None, 11, 11, 192)  6144        shuffle_12[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage3-pw-bn (BatchNormaliza (None, 11, 11, 192)  768         b5-stage3-pw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage3-pw-act (Activation)   (None, 11, 11, 192)  0           b5-stage3-pw-bn[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage3-dw (Conv2D)           (None, 11, 11, 192)  6912        b5-stage3-pw-act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage3-dw-bn (BatchNormaliza (None, 11, 11, 192)  768         b5-stage3-dw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage3-add (Add)             (None, 11, 11, 192)  0           b5-stage3-dw-bn[0][0]            \n",
      "                                                                 b5-stage2-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage3-act (Activation)      (None, 11, 11, 192)  0           b5-stage3-add[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "shuffle_13 (Shuffle)            (None, 11, 11, 192)  0           b5-stage3-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage4-pw (Conv2D)           (None, 11, 11, 192)  6144        shuffle_13[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage4-pw-bn (BatchNormaliza (None, 11, 11, 192)  768         b5-stage4-pw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage4-pw-act (Activation)   (None, 11, 11, 192)  0           b5-stage4-pw-bn[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage4-dw (Conv2D)           (None, 11, 11, 192)  6912        b5-stage4-pw-act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage4-dw-bn (BatchNormaliza (None, 11, 11, 192)  768         b5-stage4-dw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage4-add (Add)             (None, 11, 11, 192)  0           b5-stage4-dw-bn[0][0]            \n",
      "                                                                 b5-stage3-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage4-act (Activation)      (None, 11, 11, 192)  0           b5-stage4-add[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "shuffle_14 (Shuffle)            (None, 11, 11, 192)  0           b5-stage4-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage5-pw (Conv2D)           (None, 11, 11, 192)  6144        shuffle_14[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage5-pw-bn (BatchNormaliza (None, 11, 11, 192)  768         b5-stage5-pw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage5-pw-act (Activation)   (None, 11, 11, 192)  0           b5-stage5-pw-bn[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage5-dw (Conv2D)           (None, 11, 11, 192)  6912        b5-stage5-pw-act[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage5-dw-bn (BatchNormaliza (None, 11, 11, 192)  768         b5-stage5-dw[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage5-add (Add)             (None, 11, 11, 192)  0           b5-stage5-dw-bn[0][0]            \n",
      "                                                                 b5-stage4-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b5-stage5-act (Activation)      (None, 11, 11, 192)  0           b5-stage5-add[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b5-scale (Conv2D)               (None, 5, 5, 192)    13824       b5-stage5-act[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "b5-scale-bn (BatchNormalization (None, 5, 5, 192)    768         b5-scale[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "b5-scale-act (Activation)       (None, 5, 5, 192)    0           b5-scale-bn[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "b6-conv (Conv2D)                (None, 3, 3, 192)    13824       b5-scale-act[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "b6-conv-bn (BatchNormalization) (None, 3, 3, 192)    768         b6-conv[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "b6-conv-act (Activation)        (None, 3, 3, 192)    0           b6-conv-bn[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "b7-pool (GlobalAveragePooling2D (None, 192)          0           b6-conv-act[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "flatten (Flatten)               (None, 192)          0           b7-pool[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "Dense (Dense)                   (None, 40)           7720        flatten[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 209,672\n",
      "Trainable params: 7,720\n",
      "Non-trainable params: 201,952\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Setup the model\n",
    "\n",
    "model = model.make_model(input_size=385, activation=sigmoid_like, num_classes=120)\n",
    "\n",
    "# Load pretrained 120-class model\n",
    "model.load_weights('model.h5')\n",
    "\n",
    "# Make all its layers non-trainable\n",
    "for layer in model.layers:\n",
    "    layer.trainable = False\n",
    "\n",
    "# Replace last dense layer by a new one with 40 units on output\n",
    "dense_input = model.get_layer('Dense').input\n",
    "new_dense = tf.keras.layers.Dense(units=40, name='Dense')(dense_input)\n",
    "model = tf.keras.models.Model(inputs=model.input, outputs=new_dense)\n",
    "\n",
    "# Compile the model\n",
    "model.compile(optimizer=tf.keras.optimizers.Adam(0.01),\n",
    "          loss=tf.keras.losses.CategoricalCrossentropy(from_logits=True),\n",
    "          metrics=[tf.keras.metrics.CategoricalAccuracy()])\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load dataset\n",
    "image_size = model.layers[0].input_shape[0][1]\n",
    "\n",
    "(train_set, test_set), info = tfds.load(\n",
    "    'oxford_iiit_pet',\n",
    "    split=['train', 'test'],\n",
    "    shuffle_files=True,\n",
    "    as_supervised=True,\n",
    "    with_info=True)\n",
    "\n",
    "crop_size = 6 * image_size // 5\n",
    "\n",
    "train_set = train_set.shuffle(info.splits['train'].num_examples)\n",
    "train_set = train_set.map(lambda x, y: (center_crop(x, size=[crop_size, crop_size]), y))\n",
    "train_set = train_set.batch(64)\n",
    "train_set = train_set.cache()\n",
    "train_set = train_set.prefetch(AUTOTUNE)\n",
    "train_set = train_set.map(lambda x, y: augment(tf.cast(x, tf.uint8),\n",
    "                                               tf.one_hot(y, 40),\n",
    "                                               output_size=[image_size, image_size],\n",
    "                                               prescale=image_size/crop_size,\n",
    "                                               mixup=0.5,\n",
    "                                               perspective=20))\n",
    "\n",
    "test_set = test_set.map(lambda x, y: (center_crop(x, size=[image_size, image_size]) / 255,\n",
    "                                      tf.one_hot(y, 40)))\n",
    "test_set = test_set.batch(64)\n",
    "test_set = test_set.prefetch(AUTOTUNE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Sample augmented images from the training set\n",
    "it = train_set.take(1).unbatch().as_numpy_iterator()\n",
    "\n",
    "plt.figure(figsize=(12,12))\n",
    "for i in range(9):\n",
    "    plt.subplot(3, 3, i+1)\n",
    "    plt.xticks([])\n",
    "    plt.yticks([])\n",
    "    plt.grid(False)\n",
    "    image, _ = next(it)\n",
    "    plt.imshow(image)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "58/58 [==============================] - 31s 535ms/step - loss: 2.3585 - categorical_accuracy: 0.5174 - val_loss: 1.0933 - val_categorical_accuracy: 0.7817\n",
      "Epoch 2/100\n",
      "58/58 [==============================] - 26s 452ms/step - loss: 1.3466 - categorical_accuracy: 0.7481 - val_loss: 0.7210 - val_categorical_accuracy: 0.8310\n",
      "Epoch 3/100\n",
      "58/58 [==============================] - 28s 475ms/step - loss: 1.0935 - categorical_accuracy: 0.7837 - val_loss: 0.6093 - val_categorical_accuracy: 0.8422\n",
      "Epoch 4/100\n",
      "58/58 [==============================] - 29s 496ms/step - loss: 1.0391 - categorical_accuracy: 0.8071 - val_loss: 0.5620 - val_categorical_accuracy: 0.8430\n",
      "Epoch 5/100\n",
      "58/58 [==============================] - 28s 489ms/step - loss: 0.9937 - categorical_accuracy: 0.8125 - val_loss: 0.5195 - val_categorical_accuracy: 0.8479\n",
      "Epoch 6/100\n",
      "58/58 [==============================] - 29s 497ms/step - loss: 0.9400 - categorical_accuracy: 0.8321 - val_loss: 0.4920 - val_categorical_accuracy: 0.8575\n",
      "Epoch 7/100\n",
      "58/58 [==============================] - 29s 500ms/step - loss: 0.9472 - categorical_accuracy: 0.8323 - val_loss: 0.4881 - val_categorical_accuracy: 0.8528\n",
      "Epoch 8/100\n",
      "58/58 [==============================] - 29s 502ms/step - loss: 0.9161 - categorical_accuracy: 0.8356 - val_loss: 0.4741 - val_categorical_accuracy: 0.8569\n",
      "Epoch 9/100\n",
      "58/58 [==============================] - 29s 499ms/step - loss: 0.8745 - categorical_accuracy: 0.8416 - val_loss: 0.4688 - val_categorical_accuracy: 0.8547\n",
      "Epoch 10/100\n",
      "58/58 [==============================] - 29s 500ms/step - loss: 0.8898 - categorical_accuracy: 0.8451 - val_loss: 0.4618 - val_categorical_accuracy: 0.8564\n",
      "Epoch 11/100\n",
      "58/58 [==============================] - 29s 499ms/step - loss: 0.8701 - categorical_accuracy: 0.8481 - val_loss: 0.4505 - val_categorical_accuracy: 0.8615\n",
      "Epoch 12/100\n",
      "58/58 [==============================] - 29s 500ms/step - loss: 0.8643 - categorical_accuracy: 0.8470 - val_loss: 0.4511 - val_categorical_accuracy: 0.8591\n",
      "Epoch 13/100\n",
      "58/58 [==============================] - 29s 503ms/step - loss: 0.8432 - categorical_accuracy: 0.8476 - val_loss: 0.4429 - val_categorical_accuracy: 0.8626\n",
      "Epoch 14/100\n",
      "58/58 [==============================] - 29s 501ms/step - loss: 0.9055 - categorical_accuracy: 0.8446 - val_loss: 0.4490 - val_categorical_accuracy: 0.8618\n",
      "Epoch 15/100\n",
      "58/58 [==============================] - 29s 503ms/step - loss: 0.8754 - categorical_accuracy: 0.8511 - val_loss: 0.4435 - val_categorical_accuracy: 0.8637\n",
      "Epoch 16/100\n",
      "58/58 [==============================] - 29s 500ms/step - loss: 0.8479 - categorical_accuracy: 0.8649 - val_loss: 0.4498 - val_categorical_accuracy: 0.8599\n",
      "Epoch 17/100\n",
      "58/58 [==============================] - 29s 500ms/step - loss: 0.8579 - categorical_accuracy: 0.8590 - val_loss: 0.4444 - val_categorical_accuracy: 0.8602\n",
      "Epoch 18/100\n",
      "58/58 [==============================] - 29s 502ms/step - loss: 0.8472 - categorical_accuracy: 0.8568 - val_loss: 0.4378 - val_categorical_accuracy: 0.8654\n",
      "Epoch 19/100\n",
      "58/58 [==============================] - 29s 507ms/step - loss: 0.8504 - categorical_accuracy: 0.8552 - val_loss: 0.4355 - val_categorical_accuracy: 0.8632\n",
      "Epoch 20/100\n",
      "58/58 [==============================] - 29s 503ms/step - loss: 0.8470 - categorical_accuracy: 0.8573 - val_loss: 0.4336 - val_categorical_accuracy: 0.8662\n",
      "Epoch 21/100\n",
      "58/58 [==============================] - 29s 503ms/step - loss: 0.8119 - categorical_accuracy: 0.8677 - val_loss: 0.4315 - val_categorical_accuracy: 0.8684\n",
      "Epoch 22/100\n",
      "58/58 [==============================] - 29s 500ms/step - loss: 0.8003 - categorical_accuracy: 0.8747 - val_loss: 0.4316 - val_categorical_accuracy: 0.8654\n",
      "Epoch 23/100\n",
      "58/58 [==============================] - 29s 499ms/step - loss: 0.8341 - categorical_accuracy: 0.8639 - val_loss: 0.4301 - val_categorical_accuracy: 0.8640\n",
      "Epoch 24/100\n",
      "58/58 [==============================] - 29s 503ms/step - loss: 0.8070 - categorical_accuracy: 0.8696 - val_loss: 0.4279 - val_categorical_accuracy: 0.8659\n",
      "Epoch 25/100\n",
      "58/58 [==============================] - 29s 502ms/step - loss: 0.8614 - categorical_accuracy: 0.8636 - val_loss: 0.4349 - val_categorical_accuracy: 0.8648\n",
      "Epoch 26/100\n",
      "58/58 [==============================] - 29s 503ms/step - loss: 0.8226 - categorical_accuracy: 0.8628 - val_loss: 0.4327 - val_categorical_accuracy: 0.8637\n",
      "Epoch 27/100\n",
      "58/58 [==============================] - 29s 503ms/step - loss: 0.7902 - categorical_accuracy: 0.8636 - val_loss: 0.4364 - val_categorical_accuracy: 0.8637\n",
      "Epoch 28/100\n",
      "58/58 [==============================] - 29s 503ms/step - loss: 0.8366 - categorical_accuracy: 0.8582 - val_loss: 0.4325 - val_categorical_accuracy: 0.8662\n",
      "Epoch 29/100\n",
      "58/58 [==============================] - 29s 503ms/step - loss: 0.8320 - categorical_accuracy: 0.8614 - val_loss: 0.4297 - val_categorical_accuracy: 0.8678\n",
      "Epoch 30/100\n",
      "58/58 [==============================] - 29s 505ms/step - loss: 0.8208 - categorical_accuracy: 0.8628 - val_loss: 0.4341 - val_categorical_accuracy: 0.8629\n",
      "Epoch 31/100\n",
      "58/58 [==============================] - 29s 502ms/step - loss: 0.8089 - categorical_accuracy: 0.8715 - val_loss: 0.4256 - val_categorical_accuracy: 0.8664\n",
      "Epoch 32/100\n",
      "58/58 [==============================] - 29s 505ms/step - loss: 0.8241 - categorical_accuracy: 0.8598 - val_loss: 0.4258 - val_categorical_accuracy: 0.8678\n",
      "Epoch 33/100\n",
      "58/58 [==============================] - 29s 505ms/step - loss: 0.8029 - categorical_accuracy: 0.8660 - val_loss: 0.4249 - val_categorical_accuracy: 0.8684\n",
      "Epoch 34/100\n",
      "58/58 [==============================] - 29s 504ms/step - loss: 0.8356 - categorical_accuracy: 0.8541 - val_loss: 0.4282 - val_categorical_accuracy: 0.8667\n",
      "Epoch 35/100\n",
      "58/58 [==============================] - 30s 518ms/step - loss: 0.8319 - categorical_accuracy: 0.8709 - val_loss: 0.4263 - val_categorical_accuracy: 0.8673\n",
      "Epoch 36/100\n",
      "58/58 [==============================] - 30s 519ms/step - loss: 0.8220 - categorical_accuracy: 0.8698 - val_loss: 0.4261 - val_categorical_accuracy: 0.8667\n",
      "Epoch 37/100\n",
      "58/58 [==============================] - 31s 540ms/step - loss: 0.7866 - categorical_accuracy: 0.8734 - val_loss: 0.4271 - val_categorical_accuracy: 0.8670\n",
      "Epoch 38/100\n",
      "58/58 [==============================] - 30s 518ms/step - loss: 0.7919 - categorical_accuracy: 0.8736 - val_loss: 0.4267 - val_categorical_accuracy: 0.8643\n",
      "Epoch 39/100\n",
      "58/58 [==============================] - 30s 522ms/step - loss: 0.8148 - categorical_accuracy: 0.8644 - val_loss: 0.4302 - val_categorical_accuracy: 0.8664\n",
      "Epoch 40/100\n",
      "58/58 [==============================] - 30s 522ms/step - loss: 0.7895 - categorical_accuracy: 0.8774 - val_loss: 0.4259 - val_categorical_accuracy: 0.8673\n",
      "Epoch 41/100\n",
      "58/58 [==============================] - 30s 524ms/step - loss: 0.8280 - categorical_accuracy: 0.8636 - val_loss: 0.4227 - val_categorical_accuracy: 0.8673\n",
      "Epoch 42/100\n",
      "58/58 [==============================] - 30s 520ms/step - loss: 0.8132 - categorical_accuracy: 0.8709 - val_loss: 0.4226 - val_categorical_accuracy: 0.8678\n",
      "Epoch 43/100\n",
      "58/58 [==============================] - 30s 521ms/step - loss: 0.8282 - categorical_accuracy: 0.8658 - val_loss: 0.4229 - val_categorical_accuracy: 0.8694\n",
      "Epoch 44/100\n",
      "58/58 [==============================] - 30s 520ms/step - loss: 0.8012 - categorical_accuracy: 0.8758 - val_loss: 0.4222 - val_categorical_accuracy: 0.8678\n",
      "Epoch 45/100\n",
      "58/58 [==============================] - 30s 520ms/step - loss: 0.8122 - categorical_accuracy: 0.8764 - val_loss: 0.4241 - val_categorical_accuracy: 0.8675\n",
      "Epoch 46/100\n",
      "58/58 [==============================] - 30s 524ms/step - loss: 0.7965 - categorical_accuracy: 0.8764 - val_loss: 0.4246 - val_categorical_accuracy: 0.8675\n",
      "Epoch 47/100\n",
      "58/58 [==============================] - 30s 520ms/step - loss: 0.7957 - categorical_accuracy: 0.8772 - val_loss: 0.4259 - val_categorical_accuracy: 0.8670\n",
      "Epoch 48/100\n",
      "58/58 [==============================] - 30s 525ms/step - loss: 0.8085 - categorical_accuracy: 0.8726 - val_loss: 0.4243 - val_categorical_accuracy: 0.8659\n",
      "Epoch 49/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "58/58 [==============================] - 30s 520ms/step - loss: 0.7965 - categorical_accuracy: 0.8734 - val_loss: 0.4256 - val_categorical_accuracy: 0.8659\n",
      "Epoch 50/100\n",
      "58/58 [==============================] - 30s 522ms/step - loss: 0.8168 - categorical_accuracy: 0.8723 - val_loss: 0.4235 - val_categorical_accuracy: 0.8662\n",
      "Epoch 51/100\n",
      "58/58 [==============================] - 30s 526ms/step - loss: 0.7890 - categorical_accuracy: 0.8783 - val_loss: 0.4225 - val_categorical_accuracy: 0.8681\n",
      "Epoch 52/100\n",
      "58/58 [==============================] - 30s 520ms/step - loss: 0.8179 - categorical_accuracy: 0.8696 - val_loss: 0.4231 - val_categorical_accuracy: 0.8681\n",
      "Epoch 53/100\n",
      "58/58 [==============================] - 30s 524ms/step - loss: 0.7795 - categorical_accuracy: 0.8769 - val_loss: 0.4241 - val_categorical_accuracy: 0.8670\n",
      "Epoch 54/100\n",
      "58/58 [==============================] - 30s 522ms/step - loss: 0.8079 - categorical_accuracy: 0.8755 - val_loss: 0.4239 - val_categorical_accuracy: 0.8667\n",
      "Epoch 55/100\n",
      "58/58 [==============================] - 30s 519ms/step - loss: 0.8077 - categorical_accuracy: 0.8783 - val_loss: 0.4237 - val_categorical_accuracy: 0.8678\n",
      "Epoch 56/100\n",
      "58/58 [==============================] - 30s 523ms/step - loss: 0.8180 - categorical_accuracy: 0.8745 - val_loss: 0.4245 - val_categorical_accuracy: 0.8662\n",
      "Epoch 57/100\n",
      "58/58 [==============================] - 30s 522ms/step - loss: 0.7677 - categorical_accuracy: 0.8704 - val_loss: 0.4217 - val_categorical_accuracy: 0.8681\n",
      "Epoch 58/100\n",
      "58/58 [==============================] - 30s 522ms/step - loss: 0.8141 - categorical_accuracy: 0.8698 - val_loss: 0.4221 - val_categorical_accuracy: 0.8667\n",
      "Epoch 59/100\n",
      "58/58 [==============================] - 30s 521ms/step - loss: 0.8088 - categorical_accuracy: 0.8709 - val_loss: 0.4215 - val_categorical_accuracy: 0.8667\n",
      "Epoch 60/100\n",
      "58/58 [==============================] - 30s 522ms/step - loss: 0.7947 - categorical_accuracy: 0.8720 - val_loss: 0.4222 - val_categorical_accuracy: 0.8667\n",
      "Epoch 61/100\n",
      "58/58 [==============================] - 31s 526ms/step - loss: 0.8125 - categorical_accuracy: 0.8671 - val_loss: 0.4221 - val_categorical_accuracy: 0.8673\n",
      "Epoch 62/100\n",
      "58/58 [==============================] - 30s 521ms/step - loss: 0.8071 - categorical_accuracy: 0.8709 - val_loss: 0.4224 - val_categorical_accuracy: 0.8673\n",
      "Epoch 63/100\n",
      "58/58 [==============================] - 30s 521ms/step - loss: 0.7801 - categorical_accuracy: 0.8796 - val_loss: 0.4222 - val_categorical_accuracy: 0.8684\n",
      "Epoch 64/100\n",
      "58/58 [==============================] - 30s 526ms/step - loss: 0.7779 - categorical_accuracy: 0.8766 - val_loss: 0.4218 - val_categorical_accuracy: 0.8689\n",
      "Epoch 65/100\n",
      "58/58 [==============================] - 30s 525ms/step - loss: 0.8060 - categorical_accuracy: 0.8717 - val_loss: 0.4218 - val_categorical_accuracy: 0.8681\n",
      "Epoch 66/100\n",
      "58/58 [==============================] - 30s 525ms/step - loss: 0.7945 - categorical_accuracy: 0.8777 - val_loss: 0.4217 - val_categorical_accuracy: 0.8686\n",
      "Epoch 67/100\n",
      "58/58 [==============================] - 30s 524ms/step - loss: 0.8163 - categorical_accuracy: 0.8734 - val_loss: 0.4217 - val_categorical_accuracy: 0.8684\n",
      "Epoch 68/100\n",
      "58/58 [==============================] - 30s 520ms/step - loss: 0.7824 - categorical_accuracy: 0.8840 - val_loss: 0.4211 - val_categorical_accuracy: 0.8681\n",
      "Epoch 69/100\n",
      "58/58 [==============================] - 30s 521ms/step - loss: 0.7838 - categorical_accuracy: 0.8698 - val_loss: 0.4215 - val_categorical_accuracy: 0.8686\n",
      "Epoch 70/100\n",
      "58/58 [==============================] - 30s 522ms/step - loss: 0.8070 - categorical_accuracy: 0.8788 - val_loss: 0.4215 - val_categorical_accuracy: 0.8675\n",
      "Epoch 71/100\n",
      "58/58 [==============================] - 30s 525ms/step - loss: 0.8203 - categorical_accuracy: 0.8696 - val_loss: 0.4217 - val_categorical_accuracy: 0.8678\n",
      "Epoch 72/100\n",
      "58/58 [==============================] - 30s 524ms/step - loss: 0.7836 - categorical_accuracy: 0.8783 - val_loss: 0.4218 - val_categorical_accuracy: 0.8675\n",
      "Epoch 73/100\n",
      "58/58 [==============================] - 31s 526ms/step - loss: 0.7951 - categorical_accuracy: 0.8747 - val_loss: 0.4218 - val_categorical_accuracy: 0.8681\n",
      "Epoch 74/100\n",
      "58/58 [==============================] - 31s 527ms/step - loss: 0.8029 - categorical_accuracy: 0.8685 - val_loss: 0.4221 - val_categorical_accuracy: 0.8675\n",
      "Epoch 75/100\n",
      "58/58 [==============================] - 30s 525ms/step - loss: 0.8003 - categorical_accuracy: 0.8793 - val_loss: 0.4217 - val_categorical_accuracy: 0.8678\n",
      "Epoch 76/100\n",
      "58/58 [==============================] - 30s 524ms/step - loss: 0.7934 - categorical_accuracy: 0.8761 - val_loss: 0.4217 - val_categorical_accuracy: 0.8684\n",
      "Epoch 77/100\n",
      "58/58 [==============================] - 30s 525ms/step - loss: 0.8046 - categorical_accuracy: 0.8769 - val_loss: 0.4217 - val_categorical_accuracy: 0.8684\n",
      "Epoch 78/100\n",
      "58/58 [==============================] - 30s 524ms/step - loss: 0.7864 - categorical_accuracy: 0.8796 - val_loss: 0.4213 - val_categorical_accuracy: 0.8684\n",
      "Epoch 79/100\n",
      "58/58 [==============================] - 30s 522ms/step - loss: 0.8023 - categorical_accuracy: 0.8777 - val_loss: 0.4210 - val_categorical_accuracy: 0.8686\n",
      "Epoch 80/100\n",
      "58/58 [==============================] - 30s 523ms/step - loss: 0.8143 - categorical_accuracy: 0.8660 - val_loss: 0.4208 - val_categorical_accuracy: 0.8686\n",
      "Epoch 81/100\n",
      "58/58 [==============================] - 31s 528ms/step - loss: 0.7965 - categorical_accuracy: 0.8834 - val_loss: 0.4207 - val_categorical_accuracy: 0.8686\n",
      "Epoch 82/100\n",
      "58/58 [==============================] - 30s 522ms/step - loss: 0.8340 - categorical_accuracy: 0.8712 - val_loss: 0.4205 - val_categorical_accuracy: 0.8692\n",
      "Epoch 83/100\n",
      "58/58 [==============================] - 30s 520ms/step - loss: 0.8032 - categorical_accuracy: 0.8769 - val_loss: 0.4205 - val_categorical_accuracy: 0.8692\n",
      "Epoch 84/100\n",
      "58/58 [==============================] - 30s 525ms/step - loss: 0.8028 - categorical_accuracy: 0.8709 - val_loss: 0.4206 - val_categorical_accuracy: 0.8692\n",
      "Epoch 85/100\n",
      "58/58 [==============================] - 30s 521ms/step - loss: 0.7853 - categorical_accuracy: 0.8815 - val_loss: 0.4207 - val_categorical_accuracy: 0.8689\n",
      "Epoch 86/100\n",
      "58/58 [==============================] - 31s 532ms/step - loss: 0.7924 - categorical_accuracy: 0.8774 - val_loss: 0.4207 - val_categorical_accuracy: 0.8684\n",
      "Epoch 87/100\n",
      "58/58 [==============================] - 31s 532ms/step - loss: 0.8318 - categorical_accuracy: 0.8663 - val_loss: 0.4205 - val_categorical_accuracy: 0.8684\n",
      "Epoch 88/100\n",
      "58/58 [==============================] - 32s 552ms/step - loss: 0.7829 - categorical_accuracy: 0.8791 - val_loss: 0.4203 - val_categorical_accuracy: 0.8684\n",
      "Epoch 89/100\n",
      "58/58 [==============================] - 31s 530ms/step - loss: 0.8189 - categorical_accuracy: 0.8687 - val_loss: 0.4203 - val_categorical_accuracy: 0.8686\n",
      "Epoch 90/100\n",
      "58/58 [==============================] - 30s 525ms/step - loss: 0.7915 - categorical_accuracy: 0.8764 - val_loss: 0.4203 - val_categorical_accuracy: 0.8686\n",
      "Epoch 91/100\n",
      "58/58 [==============================] - 30s 519ms/step - loss: 0.8160 - categorical_accuracy: 0.8641 - val_loss: 0.4203 - val_categorical_accuracy: 0.8686\n",
      "Epoch 92/100\n",
      "58/58 [==============================] - 30s 519ms/step - loss: 0.7910 - categorical_accuracy: 0.8712 - val_loss: 0.4203 - val_categorical_accuracy: 0.8686\n",
      "Epoch 93/100\n",
      "58/58 [==============================] - 30s 521ms/step - loss: 0.7902 - categorical_accuracy: 0.8810 - val_loss: 0.4202 - val_categorical_accuracy: 0.8684\n",
      "Epoch 94/100\n",
      "58/58 [==============================] - 30s 521ms/step - loss: 0.7891 - categorical_accuracy: 0.8750 - val_loss: 0.4203 - val_categorical_accuracy: 0.8684\n",
      "Epoch 95/100\n",
      "58/58 [==============================] - 30s 522ms/step - loss: 0.7866 - categorical_accuracy: 0.8813 - val_loss: 0.4203 - val_categorical_accuracy: 0.8684\n",
      "Epoch 96/100\n",
      "58/58 [==============================] - 31s 527ms/step - loss: 0.7992 - categorical_accuracy: 0.8696 - val_loss: 0.4204 - val_categorical_accuracy: 0.8686\n",
      "Epoch 97/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "58/58 [==============================] - 30s 524ms/step - loss: 0.8393 - categorical_accuracy: 0.8690 - val_loss: 0.4204 - val_categorical_accuracy: 0.8684\n",
      "Epoch 98/100\n",
      "58/58 [==============================] - 30s 519ms/step - loss: 0.7933 - categorical_accuracy: 0.8720 - val_loss: 0.4205 - val_categorical_accuracy: 0.8684\n",
      "Epoch 99/100\n",
      "58/58 [==============================] - 30s 524ms/step - loss: 0.7884 - categorical_accuracy: 0.8666 - val_loss: 0.4205 - val_categorical_accuracy: 0.8684\n",
      "Epoch 100/100\n",
      "58/58 [==============================] - 30s 517ms/step - loss: 0.8005 - categorical_accuracy: 0.8774 - val_loss: 0.4206 - val_categorical_accuracy: 0.8684\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f1d40fc0dd8>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# FFEEEEAAAAATTTTTT\n",
    "\n",
    "# define LR schedule\n",
    "def lr_scheduler(epoch, _, initial_lr=0.1, cliff=10, step=10):\n",
    "    step = max(0, (epoch - cliff) // step + 1)\n",
    "    factor = 0.5\n",
    "    lr = initial_lr * (factor ** step)\n",
    "    return lr\n",
    "\n",
    "model.fit(train_set,\n",
    "          validation_data=test_set,\n",
    "          validation_freq=1,\n",
    "          epochs=100,\n",
    "          callbacks=[tf.keras.callbacks.LearningRateScheduler(lr_scheduler)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Run validation pass\n",
    "model.evaluate(test_set)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}