knowledge_distillation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import math\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from keras import layers, callbacks\n",
    "from keras.models import Model\n",
    "from keras.regularizers import l2\n",
    "from keras.optimizers import SGD\n",
    "from keras.applications import ResNet50V2\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.callbacks import LearningRateScheduler, EarlyStopping\n",
    "from keras.layers import Input\n",
    "from sklearn.model_selection import train_test_split\n",
    "from PIL import Image, UnidentifiedImageError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "#사용자 셋팅\n",
    "data_dir = 'TrashBox_train_set'\n",
    "saved_model_dir = 'fine_tuned_saved_model'\n",
    "saved_model_file = 'model\\ResNet50V2_fine_tuned.h5'\n",
    "image_exts = ['.jpg', '.jpeg', '.png']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 9139 validated image filenames belonging to 7 classes.\n",
      "Found 571 validated image filenames belonging to 7 classes.\n",
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " input_1 (InputLayer)           [(None, 224, 224, 3  0           []                               \n",
      "                                )]                                                                \n",
      "                                                                                                  \n",
      " conv1_pad (ZeroPadding2D)      (None, 230, 230, 3)  0           ['input_1[0][0]']                \n",
      "                                                                                                  \n",
      " conv1_conv (Conv2D)            (None, 112, 112, 64  9472        ['conv1_pad[0][0]']              \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " pool1_pad (ZeroPadding2D)      (None, 114, 114, 64  0           ['conv1_conv[0][0]']             \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " pool1_pool (MaxPooling2D)      (None, 56, 56, 64)   0           ['pool1_pad[0][0]']              \n",
      "                                                                                                  \n",
      " conv2_block1_preact_bn (BatchN  (None, 56, 56, 64)  256         ['pool1_pool[0][0]']             \n",
      " ormalization)                                                                                    \n",
      "                                                                                                  \n",
      " conv2_block1_preact_relu (Acti  (None, 56, 56, 64)  0           ['conv2_block1_preact_bn[0][0]'] \n",
      " vation)                                                                                          \n",
      "                                                                                                  \n",
      " conv2_block1_1_conv (Conv2D)   (None, 56, 56, 64)   4096        ['conv2_block1_preact_relu[0][0]'\n",
      "                                                                 ]                                \n",
      "                                                                                                  \n",
      " conv2_block1_1_bn (BatchNormal  (None, 56, 56, 64)  256         ['conv2_block1_1_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv2_block1_1_relu (Activatio  (None, 56, 56, 64)  0           ['conv2_block1_1_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv2_block1_2_pad (ZeroPaddin  (None, 58, 58, 64)  0           ['conv2_block1_1_relu[0][0]']    \n",
      " g2D)                                                                                             \n",
      "                                                                                                  \n",
      " conv2_block1_2_conv (Conv2D)   (None, 56, 56, 64)   36864       ['conv2_block1_2_pad[0][0]']     \n",
      "                                                                                                  \n",
      " conv2_block1_2_bn (BatchNormal  (None, 56, 56, 64)  256         ['conv2_block1_2_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv2_block1_2_relu (Activatio  (None, 56, 56, 64)  0           ['conv2_block1_2_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv2_block1_0_conv (Conv2D)   (None, 56, 56, 256)  16640       ['conv2_block1_preact_relu[0][0]'\n",
      "                                                                 ]                                \n",
      "                                                                                                  \n",
      " conv2_block1_3_conv (Conv2D)   (None, 56, 56, 256)  16640       ['conv2_block1_2_relu[0][0]']    \n",
      "                                                                                                  \n",
      " conv2_block1_out (Add)         (None, 56, 56, 256)  0           ['conv2_block1_0_conv[0][0]',    \n",
      "                                                                  'conv2_block1_3_conv[0][0]']    \n",
      "                                                                                                  \n",
      " conv2_block2_preact_bn (BatchN  (None, 56, 56, 256)  1024       ['conv2_block1_out[0][0]']       \n",
      " ormalization)                                                                                    \n",
      "                                                                                                  \n",
      " conv2_block2_preact_relu (Acti  (None, 56, 56, 256)  0          ['conv2_block2_preact_bn[0][0]'] \n",
      " vation)                                                                                          \n",
      "                                                                                                  \n",
      " conv2_block2_1_conv (Conv2D)   (None, 56, 56, 64)   16384       ['conv2_block2_preact_relu[0][0]'\n",
      "                                                                 ]                                \n",
      "                                                                                                  \n",
      " conv2_block2_1_bn (BatchNormal  (None, 56, 56, 64)  256         ['conv2_block2_1_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv2_block2_1_relu (Activatio  (None, 56, 56, 64)  0           ['conv2_block2_1_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv2_block2_2_pad (ZeroPaddin  (None, 58, 58, 64)  0           ['conv2_block2_1_relu[0][0]']    \n",
      " g2D)                                                                                             \n",
      "                                                                                                  \n",
      " conv2_block2_2_conv (Conv2D)   (None, 56, 56, 64)   36864       ['conv2_block2_2_pad[0][0]']     \n",
      "                                                                                                  \n",
      " conv2_block2_2_bn (BatchNormal  (None, 56, 56, 64)  256         ['conv2_block2_2_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv2_block2_2_relu (Activatio  (None, 56, 56, 64)  0           ['conv2_block2_2_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv2_block2_3_conv (Conv2D)   (None, 56, 56, 256)  16640       ['conv2_block2_2_relu[0][0]']    \n",
      "                                                                                                  \n",
      " conv2_block2_out (Add)         (None, 56, 56, 256)  0           ['conv2_block1_out[0][0]',       \n",
      "                                                                  'conv2_block2_3_conv[0][0]']    \n",
      "                                                                                                  \n",
      " conv2_block3_preact_bn (BatchN  (None, 56, 56, 256)  1024       ['conv2_block2_out[0][0]']       \n",
      " ormalization)                                                                                    \n",
      "                                                                                                  \n",
      " conv2_block3_preact_relu (Acti  (None, 56, 56, 256)  0          ['conv2_block3_preact_bn[0][0]'] \n",
      " vation)                                                                                          \n",
      "                                                                                                  \n",
      " conv2_block3_1_conv (Conv2D)   (None, 56, 56, 64)   16384       ['conv2_block3_preact_relu[0][0]'\n",
      "                                                                 ]                                \n",
      "                                                                                                  \n",
      " conv2_block3_1_bn (BatchNormal  (None, 56, 56, 64)  256         ['conv2_block3_1_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv2_block3_1_relu (Activatio  (None, 56, 56, 64)  0           ['conv2_block3_1_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv2_block3_2_pad (ZeroPaddin  (None, 58, 58, 64)  0           ['conv2_block3_1_relu[0][0]']    \n",
      " g2D)                                                                                             \n",
      "                                                                                                  \n",
      " conv2_block3_2_conv (Conv2D)   (None, 28, 28, 64)   36864       ['conv2_block3_2_pad[0][0]']     \n",
      "                                                                                                  \n",
      " conv2_block3_2_bn (BatchNormal  (None, 28, 28, 64)  256         ['conv2_block3_2_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv2_block3_2_relu (Activatio  (None, 28, 28, 64)  0           ['conv2_block3_2_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " max_pooling2d (MaxPooling2D)   (None, 28, 28, 256)  0           ['conv2_block2_out[0][0]']       \n",
      "                                                                                                  \n",
      " conv2_block3_3_conv (Conv2D)   (None, 28, 28, 256)  16640       ['conv2_block3_2_relu[0][0]']    \n",
      "                                                                                                  \n",
      " conv2_block3_out (Add)         (None, 28, 28, 256)  0           ['max_pooling2d[0][0]',          \n",
      "                                                                  'conv2_block3_3_conv[0][0]']    \n",
      "                                                                                                  \n",
      " conv3_block1_preact_bn (BatchN  (None, 28, 28, 256)  1024       ['conv2_block3_out[0][0]']       \n",
      " ormalization)                                                                                    \n",
      "                                                                                                  \n",
      " conv3_block1_preact_relu (Acti  (None, 28, 28, 256)  0          ['conv3_block1_preact_bn[0][0]'] \n",
      " vation)                                                                                          \n",
      "                                                                                                  \n",
      " conv3_block1_1_conv (Conv2D)   (None, 28, 28, 128)  32768       ['conv3_block1_preact_relu[0][0]'\n",
      "                                                                 ]                                \n",
      "                                                                                                  \n",
      " conv3_block1_1_bn (BatchNormal  (None, 28, 28, 128)  512        ['conv3_block1_1_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv3_block1_1_relu (Activatio  (None, 28, 28, 128)  0          ['conv3_block1_1_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv3_block1_2_pad (ZeroPaddin  (None, 30, 30, 128)  0          ['conv3_block1_1_relu[0][0]']    \n",
      " g2D)                                                                                             \n",
      "                                                                                                  \n",
      " conv3_block1_2_conv (Conv2D)   (None, 28, 28, 128)  147456      ['conv3_block1_2_pad[0][0]']     \n",
      "                                                                                                  \n",
      " conv3_block1_2_bn (BatchNormal  (None, 28, 28, 128)  512        ['conv3_block1_2_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv3_block1_2_relu (Activatio  (None, 28, 28, 128)  0          ['conv3_block1_2_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv3_block1_0_conv (Conv2D)   (None, 28, 28, 512)  131584      ['conv3_block1_preact_relu[0][0]'\n",
      "                                                                 ]                                \n",
      "                                                                                                  \n",
      " conv3_block1_3_conv (Conv2D)   (None, 28, 28, 512)  66048       ['conv3_block1_2_relu[0][0]']    \n",
      "                                                                                                  \n",
      " conv3_block1_out (Add)         (None, 28, 28, 512)  0           ['conv3_block1_0_conv[0][0]',    \n",
      "                                                                  'conv3_block1_3_conv[0][0]']    \n",
      "                                                                                                  \n",
      " conv3_block2_preact_bn (BatchN  (None, 28, 28, 512)  2048       ['conv3_block1_out[0][0]']       \n",
      " ormalization)                                                                                    \n",
      "                                                                                                  \n",
      " conv3_block2_preact_relu (Acti  (None, 28, 28, 512)  0          ['conv3_block2_preact_bn[0][0]'] \n",
      " vation)                                                                                          \n",
      "                                                                                                  \n",
      " conv3_block2_1_conv (Conv2D)   (None, 28, 28, 128)  65536       ['conv3_block2_preact_relu[0][0]'\n",
      "                                                                 ]                                \n",
      "                                                                                                  \n",
      " conv3_block2_1_bn (BatchNormal  (None, 28, 28, 128)  512        ['conv3_block2_1_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv3_block2_1_relu (Activatio  (None, 28, 28, 128)  0          ['conv3_block2_1_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv3_block2_2_pad (ZeroPaddin  (None, 30, 30, 128)  0          ['conv3_block2_1_relu[0][0]']    \n",
      " g2D)                                                                                             \n",
      "                                                                                                  \n",
      " conv3_block2_2_conv (Conv2D)   (None, 28, 28, 128)  147456      ['conv3_block2_2_pad[0][0]']     \n",
      "                                                                                                  \n",
      " conv3_block2_2_bn (BatchNormal  (None, 28, 28, 128)  512        ['conv3_block2_2_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv3_block2_2_relu (Activatio  (None, 28, 28, 128)  0          ['conv3_block2_2_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv3_block2_3_conv (Conv2D)   (None, 28, 28, 512)  66048       ['conv3_block2_2_relu[0][0]']    \n",
      "                                                                                                  \n",
      " conv3_block2_out (Add)         (None, 28, 28, 512)  0           ['conv3_block1_out[0][0]',       \n",
      "                                                                  'conv3_block2_3_conv[0][0]']    \n",
      "                                                                                                  \n",
      " conv3_block3_preact_bn (BatchN  (None, 28, 28, 512)  2048       ['conv3_block2_out[0][0]']       \n",
      " ormalization)                                                                                    \n",
      "                                                                                                  \n",
      " conv3_block3_preact_relu (Acti  (None, 28, 28, 512)  0          ['conv3_block3_preact_bn[0][0]'] \n",
      " vation)                                                                                          \n",
      "                                                                                                  \n",
      " conv3_block3_1_conv (Conv2D)   (None, 28, 28, 128)  65536       ['conv3_block3_preact_relu[0][0]'\n",
      "                                                                 ]                                \n",
      "                                                                                                  \n",
      " conv3_block3_1_bn (BatchNormal  (None, 28, 28, 128)  512        ['conv3_block3_1_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv3_block3_1_relu (Activatio  (None, 28, 28, 128)  0          ['conv3_block3_1_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv3_block3_2_pad (ZeroPaddin  (None, 30, 30, 128)  0          ['conv3_block3_1_relu[0][0]']    \n",
      " g2D)                                                                                             \n",
      "                                                                                                  \n",
      " conv3_block3_2_conv (Conv2D)   (None, 28, 28, 128)  147456      ['conv3_block3_2_pad[0][0]']     \n",
      "                                                                                                  \n",
      " conv3_block3_2_bn (BatchNormal  (None, 28, 28, 128)  512        ['conv3_block3_2_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv3_block3_2_relu (Activatio  (None, 28, 28, 128)  0          ['conv3_block3_2_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv3_block3_3_conv (Conv2D)   (None, 28, 28, 512)  66048       ['conv3_block3_2_relu[0][0]']    \n",
      "                                                                                                  \n",
      " conv3_block3_out (Add)         (None, 28, 28, 512)  0           ['conv3_block2_out[0][0]',       \n",
      "                                                                  'conv3_block3_3_conv[0][0]']    \n",
      "                                                                                                  \n",
      " conv3_block4_preact_bn (BatchN  (None, 28, 28, 512)  2048       ['conv3_block3_out[0][0]']       \n",
      " ormalization)                                                                                    \n",
      "                                                                                                  \n",
      " conv3_block4_preact_relu (Acti  (None, 28, 28, 512)  0          ['conv3_block4_preact_bn[0][0]'] \n",
      " vation)                                                                                          \n",
      "                                                                                                  \n",
      " conv3_block4_1_conv (Conv2D)   (None, 28, 28, 128)  65536       ['conv3_block4_preact_relu[0][0]'\n",
      "                                                                 ]                                \n",
      "                                                                                                  \n",
      " conv3_block4_1_bn (BatchNormal  (None, 28, 28, 128)  512        ['conv3_block4_1_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv3_block4_1_relu (Activatio  (None, 28, 28, 128)  0          ['conv3_block4_1_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv3_block4_2_pad (ZeroPaddin  (None, 30, 30, 128)  0          ['conv3_block4_1_relu[0][0]']    \n",
      " g2D)                                                                                             \n",
      "                                                                                                  \n",
      " conv3_block4_2_conv (Conv2D)   (None, 14, 14, 128)  147456      ['conv3_block4_2_pad[0][0]']     \n",
      "                                                                                                  \n",
      " conv3_block4_2_bn (BatchNormal  (None, 14, 14, 128)  512        ['conv3_block4_2_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv3_block4_2_relu (Activatio  (None, 14, 14, 128)  0          ['conv3_block4_2_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " max_pooling2d_1 (MaxPooling2D)  (None, 14, 14, 512)  0          ['conv3_block3_out[0][0]']       \n",
      "                                                                                                  \n",
      " conv3_block4_3_conv (Conv2D)   (None, 14, 14, 512)  66048       ['conv3_block4_2_relu[0][0]']    \n",
      "                                                                                                  \n",
      " conv3_block4_out (Add)         (None, 14, 14, 512)  0           ['max_pooling2d_1[0][0]',        \n",
      "                                                                  'conv3_block4_3_conv[0][0]']    \n",
      "                                                                                                  \n",
      " conv4_block1_preact_bn (BatchN  (None, 14, 14, 512)  2048       ['conv3_block4_out[0][0]']       \n",
      " ormalization)                                                                                    \n",
      "                                                                                                  \n",
      " conv4_block1_preact_relu (Acti  (None, 14, 14, 512)  0          ['conv4_block1_preact_bn[0][0]'] \n",
      " vation)                                                                                          \n",
      "                                                                                                  \n",
      " conv4_block1_1_conv (Conv2D)   (None, 14, 14, 256)  131072      ['conv4_block1_preact_relu[0][0]'\n",
      "                                                                 ]                                \n",
      "                                                                                                  \n",
      " conv4_block1_1_bn (BatchNormal  (None, 14, 14, 256)  1024       ['conv4_block1_1_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv4_block1_1_relu (Activatio  (None, 14, 14, 256)  0          ['conv4_block1_1_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv4_block1_2_pad (ZeroPaddin  (None, 16, 16, 256)  0          ['conv4_block1_1_relu[0][0]']    \n",
      " g2D)                                                                                             \n",
      "                                                                                                  \n",
      " conv4_block1_2_conv (Conv2D)   (None, 14, 14, 256)  589824      ['conv4_block1_2_pad[0][0]']     \n",
      "                                                                                                  \n",
      " conv4_block1_2_bn (BatchNormal  (None, 14, 14, 256)  1024       ['conv4_block1_2_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv4_block1_2_relu (Activatio  (None, 14, 14, 256)  0          ['conv4_block1_2_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv4_block1_0_conv (Conv2D)   (None, 14, 14, 1024  525312      ['conv4_block1_preact_relu[0][0]'\n",
      "                                )                                ]                                \n",
      "                                                                                                  \n",
      " conv4_block1_3_conv (Conv2D)   (None, 14, 14, 1024  263168      ['conv4_block1_2_relu[0][0]']    \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " conv4_block1_out (Add)         (None, 14, 14, 1024  0           ['conv4_block1_0_conv[0][0]',    \n",
      "                                )                                 'conv4_block1_3_conv[0][0]']    \n",
      "                                                                                                  \n",
      " conv4_block2_preact_bn (BatchN  (None, 14, 14, 1024  4096       ['conv4_block1_out[0][0]']       \n",
      " ormalization)                  )                                                                 \n",
      "                                                                                                  \n",
      " conv4_block2_preact_relu (Acti  (None, 14, 14, 1024  0          ['conv4_block2_preact_bn[0][0]'] \n",
      " vation)                        )                                                                 \n",
      "                                                                                                  \n",
      " conv4_block2_1_conv (Conv2D)   (None, 14, 14, 256)  262144      ['conv4_block2_preact_relu[0][0]'\n",
      "                                                                 ]                                \n",
      "                                                                                                  \n",
      " conv4_block2_1_bn (BatchNormal  (None, 14, 14, 256)  1024       ['conv4_block2_1_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv4_block2_1_relu (Activatio  (None, 14, 14, 256)  0          ['conv4_block2_1_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv4_block2_2_pad (ZeroPaddin  (None, 16, 16, 256)  0          ['conv4_block2_1_relu[0][0]']    \n",
      " g2D)                                                                                             \n",
      "                                                                                                  \n",
      " conv4_block2_2_conv (Conv2D)   (None, 14, 14, 256)  589824      ['conv4_block2_2_pad[0][0]']     \n",
      "                                                                                                  \n",
      " conv4_block2_2_bn (BatchNormal  (None, 14, 14, 256)  1024       ['conv4_block2_2_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv4_block2_2_relu (Activatio  (None, 14, 14, 256)  0          ['conv4_block2_2_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv4_block2_3_conv (Conv2D)   (None, 14, 14, 1024  263168      ['conv4_block2_2_relu[0][0]']    \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " conv4_block2_out (Add)         (None, 14, 14, 1024  0           ['conv4_block1_out[0][0]',       \n",
      "                                )                                 'conv4_block2_3_conv[0][0]']    \n",
      "                                                                                                  \n",
      " conv4_block3_preact_bn (BatchN  (None, 14, 14, 1024  4096       ['conv4_block2_out[0][0]']       \n",
      " ormalization)                  )                                                                 \n",
      "                                                                                                  \n",
      " conv4_block3_preact_relu (Acti  (None, 14, 14, 1024  0          ['conv4_block3_preact_bn[0][0]'] \n",
      " vation)                        )                                                                 \n",
      "                                                                                                  \n",
      " conv4_block3_1_conv (Conv2D)   (None, 14, 14, 256)  262144      ['conv4_block3_preact_relu[0][0]'\n",
      "                                                                 ]                                \n",
      "                                                                                                  \n",
      " conv4_block3_1_bn (BatchNormal  (None, 14, 14, 256)  1024       ['conv4_block3_1_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv4_block3_1_relu (Activatio  (None, 14, 14, 256)  0          ['conv4_block3_1_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv4_block3_2_pad (ZeroPaddin  (None, 16, 16, 256)  0          ['conv4_block3_1_relu[0][0]']    \n",
      " g2D)                                                                                             \n",
      "                                                                                                  \n",
      " conv4_block3_2_conv (Conv2D)   (None, 14, 14, 256)  589824      ['conv4_block3_2_pad[0][0]']     \n",
      "                                                                                                  \n",
      " conv4_block3_2_bn (BatchNormal  (None, 14, 14, 256)  1024       ['conv4_block3_2_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv4_block3_2_relu (Activatio  (None, 14, 14, 256)  0          ['conv4_block3_2_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv4_block3_3_conv (Conv2D)   (None, 14, 14, 1024  263168      ['conv4_block3_2_relu[0][0]']    \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " conv4_block3_out (Add)         (None, 14, 14, 1024  0           ['conv4_block2_out[0][0]',       \n",
      "                                )                                 'conv4_block3_3_conv[0][0]']    \n",
      "                                                                                                  \n",
      " conv4_block4_preact_bn (BatchN  (None, 14, 14, 1024  4096       ['conv4_block3_out[0][0]']       \n",
      " ormalization)                  )                                                                 \n",
      "                                                                                                  \n",
      " conv4_block4_preact_relu (Acti  (None, 14, 14, 1024  0          ['conv4_block4_preact_bn[0][0]'] \n",
      " vation)                        )                                                                 \n",
      "                                                                                                  \n",
      " conv4_block4_1_conv (Conv2D)   (None, 14, 14, 256)  262144      ['conv4_block4_preact_relu[0][0]'\n",
      "                                                                 ]                                \n",
      "                                                                                                  \n",
      " conv4_block4_1_bn (BatchNormal  (None, 14, 14, 256)  1024       ['conv4_block4_1_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv4_block4_1_relu (Activatio  (None, 14, 14, 256)  0          ['conv4_block4_1_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv4_block4_2_pad (ZeroPaddin  (None, 16, 16, 256)  0          ['conv4_block4_1_relu[0][0]']    \n",
      " g2D)                                                                                             \n",
      "                                                                                                  \n",
      " conv4_block4_2_conv (Conv2D)   (None, 14, 14, 256)  589824      ['conv4_block4_2_pad[0][0]']     \n",
      "                                                                                                  \n",
      " conv4_block4_2_bn (BatchNormal  (None, 14, 14, 256)  1024       ['conv4_block4_2_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv4_block4_2_relu (Activatio  (None, 14, 14, 256)  0          ['conv4_block4_2_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv4_block4_3_conv (Conv2D)   (None, 14, 14, 1024  263168      ['conv4_block4_2_relu[0][0]']    \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " conv4_block4_out (Add)         (None, 14, 14, 1024  0           ['conv4_block3_out[0][0]',       \n",
      "                                )                                 'conv4_block4_3_conv[0][0]']    \n",
      "                                                                                                  \n",
      " conv4_block5_preact_bn (BatchN  (None, 14, 14, 1024  4096       ['conv4_block4_out[0][0]']       \n",
      " ormalization)                  )                                                                 \n",
      "                                                                                                  \n",
      " conv4_block5_preact_relu (Acti  (None, 14, 14, 1024  0          ['conv4_block5_preact_bn[0][0]'] \n",
      " vation)                        )                                                                 \n",
      "                                                                                                  \n",
      " conv4_block5_1_conv (Conv2D)   (None, 14, 14, 256)  262144      ['conv4_block5_preact_relu[0][0]'\n",
      "                                                                 ]                                \n",
      "                                                                                                  \n",
      " conv4_block5_1_bn (BatchNormal  (None, 14, 14, 256)  1024       ['conv4_block5_1_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv4_block5_1_relu (Activatio  (None, 14, 14, 256)  0          ['conv4_block5_1_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv4_block5_2_pad (ZeroPaddin  (None, 16, 16, 256)  0          ['conv4_block5_1_relu[0][0]']    \n",
      " g2D)                                                                                             \n",
      "                                                                                                  \n",
      " conv4_block5_2_conv (Conv2D)   (None, 14, 14, 256)  589824      ['conv4_block5_2_pad[0][0]']     \n",
      "                                                                                                  \n",
      " conv4_block5_2_bn (BatchNormal  (None, 14, 14, 256)  1024       ['conv4_block5_2_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv4_block5_2_relu (Activatio  (None, 14, 14, 256)  0          ['conv4_block5_2_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv4_block5_3_conv (Conv2D)   (None, 14, 14, 1024  263168      ['conv4_block5_2_relu[0][0]']    \n",
      "                                )                                                                 \n",
      "                                                                                                  \n",
      " conv4_block5_out (Add)         (None, 14, 14, 1024  0           ['conv4_block4_out[0][0]',       \n",
      "                                )                                 'conv4_block5_3_conv[0][0]']    \n",
      "                                                                                                  \n",
      " conv4_block6_preact_bn (BatchN  (None, 14, 14, 1024  4096       ['conv4_block5_out[0][0]']       \n",
      " ormalization)                  )                                                                 \n",
      "                                                                                                  \n",
      " conv4_block6_preact_relu (Acti  (None, 14, 14, 1024  0          ['conv4_block6_preact_bn[0][0]'] \n",
      " vation)                        )                                                                 \n",
      "                                                                                                  \n",
      " conv4_block6_1_conv (Conv2D)   (None, 14, 14, 256)  262144      ['conv4_block6_preact_relu[0][0]'\n",
      "                                                                 ]                                \n",
      "                                                                                                  \n",
      " conv4_block6_1_bn (BatchNormal  (None, 14, 14, 256)  1024       ['conv4_block6_1_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv4_block6_1_relu (Activatio  (None, 14, 14, 256)  0          ['conv4_block6_1_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv4_block6_2_pad (ZeroPaddin  (None, 16, 16, 256)  0          ['conv4_block6_1_relu[0][0]']    \n",
      " g2D)                                                                                             \n",
      "                                                                                                  \n",
      " conv4_block6_2_conv (Conv2D)   (None, 7, 7, 256)    589824      ['conv4_block6_2_pad[0][0]']     \n",
      "                                                                                                  \n",
      " conv4_block6_2_bn (BatchNormal  (None, 7, 7, 256)   1024        ['conv4_block6_2_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv4_block6_2_relu (Activatio  (None, 7, 7, 256)   0           ['conv4_block6_2_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " max_pooling2d_2 (MaxPooling2D)  (None, 7, 7, 1024)  0           ['conv4_block5_out[0][0]']       \n",
      "                                                                                                  \n",
      " conv4_block6_3_conv (Conv2D)   (None, 7, 7, 1024)   263168      ['conv4_block6_2_relu[0][0]']    \n",
      "                                                                                                  \n",
      " conv4_block6_out (Add)         (None, 7, 7, 1024)   0           ['max_pooling2d_2[0][0]',        \n",
      "                                                                  'conv4_block6_3_conv[0][0]']    \n",
      "                                                                                                  \n",
      " conv5_block1_preact_bn (BatchN  (None, 7, 7, 1024)  4096        ['conv4_block6_out[0][0]']       \n",
      " ormalization)                                                                                    \n",
      "                                                                                                  \n",
      " conv5_block1_preact_relu (Acti  (None, 7, 7, 1024)  0           ['conv5_block1_preact_bn[0][0]'] \n",
      " vation)                                                                                          \n",
      "                                                                                                  \n",
      " conv5_block1_1_conv (Conv2D)   (None, 7, 7, 512)    524288      ['conv5_block1_preact_relu[0][0]'\n",
      "                                                                 ]                                \n",
      "                                                                                                  \n",
      " conv5_block1_1_bn (BatchNormal  (None, 7, 7, 512)   2048        ['conv5_block1_1_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv5_block1_1_relu (Activatio  (None, 7, 7, 512)   0           ['conv5_block1_1_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv5_block1_2_pad (ZeroPaddin  (None, 9, 9, 512)   0           ['conv5_block1_1_relu[0][0]']    \n",
      " g2D)                                                                                             \n",
      "                                                                                                  \n",
      " conv5_block1_2_conv (Conv2D)   (None, 7, 7, 512)    2359296     ['conv5_block1_2_pad[0][0]']     \n",
      "                                                                                                  \n",
      " conv5_block1_2_bn (BatchNormal  (None, 7, 7, 512)   2048        ['conv5_block1_2_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv5_block1_2_relu (Activatio  (None, 7, 7, 512)   0           ['conv5_block1_2_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv5_block1_0_conv (Conv2D)   (None, 7, 7, 2048)   2099200     ['conv5_block1_preact_relu[0][0]'\n",
      "                                                                 ]                                \n",
      "                                                                                                  \n",
      " conv5_block1_3_conv (Conv2D)   (None, 7, 7, 2048)   1050624     ['conv5_block1_2_relu[0][0]']    \n",
      "                                                                                                  \n",
      " conv5_block1_out (Add)         (None, 7, 7, 2048)   0           ['conv5_block1_0_conv[0][0]',    \n",
      "                                                                  'conv5_block1_3_conv[0][0]']    \n",
      "                                                                                                  \n",
      " conv5_block2_preact_bn (BatchN  (None, 7, 7, 2048)  8192        ['conv5_block1_out[0][0]']       \n",
      " ormalization)                                                                                    \n",
      "                                                                                                  \n",
      " conv5_block2_preact_relu (Acti  (None, 7, 7, 2048)  0           ['conv5_block2_preact_bn[0][0]'] \n",
      " vation)                                                                                          \n",
      "                                                                                                  \n",
      " conv5_block2_1_conv (Conv2D)   (None, 7, 7, 512)    1048576     ['conv5_block2_preact_relu[0][0]'\n",
      "                                                                 ]                                \n",
      "                                                                                                  \n",
      " conv5_block2_1_bn (BatchNormal  (None, 7, 7, 512)   2048        ['conv5_block2_1_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv5_block2_1_relu (Activatio  (None, 7, 7, 512)   0           ['conv5_block2_1_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv5_block2_2_pad (ZeroPaddin  (None, 9, 9, 512)   0           ['conv5_block2_1_relu[0][0]']    \n",
      " g2D)                                                                                             \n",
      "                                                                                                  \n",
      " conv5_block2_2_conv (Conv2D)   (None, 7, 7, 512)    2359296     ['conv5_block2_2_pad[0][0]']     \n",
      "                                                                                                  \n",
      " conv5_block2_2_bn (BatchNormal  (None, 7, 7, 512)   2048        ['conv5_block2_2_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv5_block2_2_relu (Activatio  (None, 7, 7, 512)   0           ['conv5_block2_2_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv5_block2_3_conv (Conv2D)   (None, 7, 7, 2048)   1050624     ['conv5_block2_2_relu[0][0]']    \n",
      "                                                                                                  \n",
      " conv5_block2_out (Add)         (None, 7, 7, 2048)   0           ['conv5_block1_out[0][0]',       \n",
      "                                                                  'conv5_block2_3_conv[0][0]']    \n",
      "                                                                                                  \n",
      " conv5_block3_preact_bn (BatchN  (None, 7, 7, 2048)  8192        ['conv5_block2_out[0][0]']       \n",
      " ormalization)                                                                                    \n",
      "                                                                                                  \n",
      " conv5_block3_preact_relu (Acti  (None, 7, 7, 2048)  0           ['conv5_block3_preact_bn[0][0]'] \n",
      " vation)                                                                                          \n",
      "                                                                                                  \n",
      " conv5_block3_1_conv (Conv2D)   (None, 7, 7, 512)    1048576     ['conv5_block3_preact_relu[0][0]'\n",
      "                                                                 ]                                \n",
      "                                                                                                  \n",
      " conv5_block3_1_bn (BatchNormal  (None, 7, 7, 512)   2048        ['conv5_block3_1_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv5_block3_1_relu (Activatio  (None, 7, 7, 512)   0           ['conv5_block3_1_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv5_block3_2_pad (ZeroPaddin  (None, 9, 9, 512)   0           ['conv5_block3_1_relu[0][0]']    \n",
      " g2D)                                                                                             \n",
      "                                                                                                  \n",
      " conv5_block3_2_conv (Conv2D)   (None, 7, 7, 512)    2359296     ['conv5_block3_2_pad[0][0]']     \n",
      "                                                                                                  \n",
      " conv5_block3_2_bn (BatchNormal  (None, 7, 7, 512)   2048        ['conv5_block3_2_conv[0][0]']    \n",
      " ization)                                                                                         \n",
      "                                                                                                  \n",
      " conv5_block3_2_relu (Activatio  (None, 7, 7, 512)   0           ['conv5_block3_2_bn[0][0]']      \n",
      " n)                                                                                               \n",
      "                                                                                                  \n",
      " conv5_block3_3_conv (Conv2D)   (None, 7, 7, 2048)   1050624     ['conv5_block3_2_relu[0][0]']    \n",
      "                                                                                                  \n",
      " conv5_block3_out (Add)         (None, 7, 7, 2048)   0           ['conv5_block2_out[0][0]',       \n",
      "                                                                  'conv5_block3_3_conv[0][0]']    \n",
      "                                                                                                  \n",
      " post_bn (BatchNormalization)   (None, 7, 7, 2048)   8192        ['conv5_block3_out[0][0]']       \n",
      "                                                                                                  \n",
      " post_relu (Activation)         (None, 7, 7, 2048)   0           ['post_bn[0][0]']                \n",
      "                                                                                                  \n",
      " global_average_pooling2d (Glob  (None, 2048)        0           ['post_relu[0][0]']              \n",
      " alAveragePooling2D)                                                                              \n",
      "                                                                                                  \n",
      " dense (Dense)                  (None, 1024)         2098176     ['global_average_pooling2d[0][0]'\n",
      "                                                                 ]                                \n",
      "                                                                                                  \n",
      " batch_normalization (BatchNorm  (None, 1024)        4096        ['dense[0][0]']                  \n",
      " alization)                                                                                       \n",
      "                                                                                                  \n",
      " activation (Activation)        (None, 1024)         0           ['batch_normalization[0][0]']    \n",
      "                                                                                                  \n",
      " dropout (Dropout)              (None, 1024)         0           ['activation[0][0]']             \n",
      "                                                                                                  \n",
      " dense_1 (Dense)                (None, 7)            7175        ['dropout[0][0]']                \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 25,674,247\n",
      "Trainable params: 25,626,759\n",
      "Non-trainable params: 47,488\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\layen\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\gradient_descent.py:108: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(SGD, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\layen\\anaconda3\\lib\\site-packages\\PIL\\Image.py:996: UserWarning: Palette images with Transparency expressed in bytes should be converted to RGBA images\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "286/286 [==============================] - 232s 783ms/step - loss: 8.4181 - accuracy: 0.5690 - val_loss: 5.5130 - val_accuracy: 0.7303 - lr: 0.1000\n",
      "Epoch 2/30\n",
      "286/286 [==============================] - 149s 520ms/step - loss: 10.1317 - accuracy: 0.6184 - val_loss: 6.4134 - val_accuracy: 0.7233 - lr: 0.1000\n",
      "Epoch 3/30\n",
      "286/286 [==============================] - 151s 526ms/step - loss: 9.8666 - accuracy: 0.6426 - val_loss: 6.1748 - val_accuracy: 0.7373 - lr: 0.1000\n",
      "Epoch 4/30\n",
      "286/286 [==============================] - 222s 777ms/step - loss: 9.8840 - accuracy: 0.6477 - val_loss: 6.3452 - val_accuracy: 0.7163 - lr: 0.1000\n",
      "Epoch 5/30\n",
      "286/286 [==============================] - 150s 525ms/step - loss: 10.2765 - accuracy: 0.6522 - val_loss: 5.9572 - val_accuracy: 0.7461 - lr: 0.1000\n",
      "Epoch 6/30\n",
      "286/286 [==============================] - 149s 520ms/step - loss: 10.2621 - accuracy: 0.6583 - val_loss: 5.5446 - val_accuracy: 0.7566 - lr: 0.1000\n",
      "Epoch 7/30\n",
      "286/286 [==============================] - 149s 521ms/step - loss: 9.9842 - accuracy: 0.6605 - val_loss: 5.5611 - val_accuracy: 0.7513 - lr: 0.1000\n",
      "Epoch 8/30\n",
      "286/286 [==============================] - 152s 530ms/step - loss: 9.8469 - accuracy: 0.6619 - val_loss: 5.7440 - val_accuracy: 0.7566 - lr: 0.1000\n",
      "Epoch 9/30\n",
      "286/286 [==============================] - 149s 519ms/step - loss: 9.6414 - accuracy: 0.6642 - val_loss: 5.3910 - val_accuracy: 0.7548 - lr: 0.1000\n",
      "Epoch 10/30\n",
      "286/286 [==============================] - 150s 524ms/step - loss: 8.2058 - accuracy: 0.6853 - val_loss: 4.9300 - val_accuracy: 0.7566 - lr: 0.0500\n",
      "Epoch 11/30\n",
      "286/286 [==============================] - 150s 523ms/step - loss: 6.7885 - accuracy: 0.7037 - val_loss: 3.8644 - val_accuracy: 0.7828 - lr: 0.0500\n",
      "Epoch 12/30\n",
      "286/286 [==============================] - 149s 523ms/step - loss: 6.2191 - accuracy: 0.7030 - val_loss: 4.0743 - val_accuracy: 0.7776 - lr: 0.0500\n",
      "Epoch 13/30\n",
      "286/286 [==============================] - 149s 521ms/step - loss: 6.2284 - accuracy: 0.6814 - val_loss: 4.0588 - val_accuracy: 0.7443 - lr: 0.0500\n",
      "Epoch 14/30\n",
      "286/286 [==============================] - 149s 521ms/step - loss: 5.8069 - accuracy: 0.6886 - val_loss: 3.7818 - val_accuracy: 0.7636 - lr: 0.0500\n",
      "Epoch 15/30\n",
      "286/286 [==============================] - 148s 517ms/step - loss: 5.6899 - accuracy: 0.6860 - val_loss: 3.5481 - val_accuracy: 0.7758 - lr: 0.0500\n",
      "Epoch 16/30\n",
      "286/286 [==============================] - 148s 518ms/step - loss: 5.6306 - accuracy: 0.6821 - val_loss: 3.4793 - val_accuracy: 0.7478 - lr: 0.0500\n",
      "Epoch 17/30\n",
      "286/286 [==============================] - 149s 520ms/step - loss: 5.5040 - accuracy: 0.6796 - val_loss: 2.7820 - val_accuracy: 0.7933 - lr: 0.0500\n",
      "Epoch 18/30\n",
      "286/286 [==============================] - 148s 519ms/step - loss: 5.1860 - accuracy: 0.6891 - val_loss: 3.2039 - val_accuracy: 0.7531 - lr: 0.0500\n",
      "Epoch 19/30\n",
      "286/286 [==============================] - 149s 520ms/step - loss: 5.2632 - accuracy: 0.6783 - val_loss: 2.8203 - val_accuracy: 0.7846 - lr: 0.0500\n",
      "Epoch 20/30\n",
      "286/286 [==============================] - 149s 521ms/step - loss: 4.6874 - accuracy: 0.6985 - val_loss: 2.4048 - val_accuracy: 0.7898 - lr: 0.0250\n",
      "Epoch 21/30\n",
      "286/286 [==============================] - 149s 520ms/step - loss: 3.8771 - accuracy: 0.7077 - val_loss: 2.2360 - val_accuracy: 0.7863 - lr: 0.0250\n",
      "Epoch 22/30\n",
      "286/286 [==============================] - 149s 521ms/step - loss: 3.6594 - accuracy: 0.7096 - val_loss: 2.3329 - val_accuracy: 0.7776 - lr: 0.0250\n",
      "Epoch 23/30\n",
      "286/286 [==============================] - 148s 518ms/step - loss: 3.5462 - accuracy: 0.7019 - val_loss: 2.3688 - val_accuracy: 0.7566 - lr: 0.0250\n",
      "Epoch 24/30\n",
      "286/286 [==============================] - 148s 518ms/step - loss: 3.2674 - accuracy: 0.7066 - val_loss: 2.2341 - val_accuracy: 0.7496 - lr: 0.0250\n",
      "Epoch 25/30\n",
      "286/286 [==============================] - 149s 521ms/step - loss: 3.1139 - accuracy: 0.7025 - val_loss: 1.9270 - val_accuracy: 0.7776 - lr: 0.0250\n",
      "Epoch 26/30\n",
      "286/286 [==============================] - 149s 521ms/step - loss: 3.2093 - accuracy: 0.6959 - val_loss: 1.8586 - val_accuracy: 0.7898 - lr: 0.0250\n",
      "Epoch 27/30\n",
      "286/286 [==============================] - 149s 522ms/step - loss: 3.0845 - accuracy: 0.6900 - val_loss: 2.2444 - val_accuracy: 0.7478 - lr: 0.0250\n",
      "Epoch 28/30\n",
      "286/286 [==============================] - 150s 523ms/step - loss: 3.0560 - accuracy: 0.6927 - val_loss: 2.1328 - val_accuracy: 0.7461 - lr: 0.0250\n",
      "Epoch 29/30\n",
      "286/286 [==============================] - 149s 522ms/step - loss: 2.9064 - accuracy: 0.6967 - val_loss: 2.1875 - val_accuracy: 0.7513 - lr: 0.0250\n",
      "Epoch 30/30\n",
      "286/286 [==============================] - 150s 523ms/step - loss: 2.5946 - accuracy: 0.7095 - val_loss: 1.6104 - val_accuracy: 0.7741 - lr: 0.0125\n",
      "Epoch 1/30\n",
      "286/286 [==============================] - 153s 526ms/step - loss: 7.4438 - accuracy: 0.6197 - val_loss: 5.3028 - val_accuracy: 0.7285 - lr: 0.1000\n",
      "Epoch 2/30\n",
      "286/286 [==============================] - 151s 529ms/step - loss: 9.0283 - accuracy: 0.6410 - val_loss: 5.6867 - val_accuracy: 0.7250 - lr: 0.1000\n",
      "Epoch 3/30\n",
      "286/286 [==============================] - 149s 522ms/step - loss: 9.2311 - accuracy: 0.6522 - val_loss: 5.6375 - val_accuracy: 0.7128 - lr: 0.1000\n",
      "Epoch 4/30\n",
      "286/286 [==============================] - 150s 524ms/step - loss: 9.5190 - accuracy: 0.6619 - val_loss: 7.1312 - val_accuracy: 0.7303 - lr: 0.1000\n",
      "Epoch 5/30\n",
      "286/286 [==============================] - 150s 524ms/step - loss: 9.7319 - accuracy: 0.6553 - val_loss: 5.7544 - val_accuracy: 0.7356 - lr: 0.1000\n",
      "Epoch 6/30\n",
      "286/286 [==============================] - 149s 521ms/step - loss: 9.7661 - accuracy: 0.6616 - val_loss: 6.0943 - val_accuracy: 0.7478 - lr: 0.1000\n",
      "Epoch 7/30\n",
      "286/286 [==============================] - 152s 531ms/step - loss: 10.1483 - accuracy: 0.6506 - val_loss: 5.7140 - val_accuracy: 0.7320 - lr: 0.1000\n",
      "Epoch 8/30\n",
      "286/286 [==============================] - 150s 524ms/step - loss: 9.4571 - accuracy: 0.6693 - val_loss: 6.4973 - val_accuracy: 0.7356 - lr: 0.1000\n",
      "Epoch 9/30\n",
      "286/286 [==============================] - 149s 521ms/step - loss: 10.2015 - accuracy: 0.6541 - val_loss: 6.4260 - val_accuracy: 0.7408 - lr: 0.1000\n",
      "Epoch 10/30\n",
      "286/286 [==============================] - 154s 537ms/step - loss: 8.2173 - accuracy: 0.6820 - val_loss: 4.2936 - val_accuracy: 0.7758 - lr: 0.0500\n",
      "Epoch 11/30\n",
      "286/286 [==============================] - 150s 526ms/step - loss: 7.2376 - accuracy: 0.6892 - val_loss: 4.2952 - val_accuracy: 0.7548 - lr: 0.0500\n",
      "Epoch 12/30\n",
      "286/286 [==============================] - 149s 522ms/step - loss: 6.5782 - accuracy: 0.6881 - val_loss: 3.8398 - val_accuracy: 0.7636 - lr: 0.0500\n",
      "Epoch 13/30\n",
      "286/286 [==============================] - 150s 524ms/step - loss: 5.8842 - accuracy: 0.6970 - val_loss: 3.1004 - val_accuracy: 0.7881 - lr: 0.0500\n",
      "Epoch 14/30\n",
      "286/286 [==============================] - 150s 526ms/step - loss: 5.8014 - accuracy: 0.6878 - val_loss: 3.0815 - val_accuracy: 0.7793 - lr: 0.0500\n",
      "Epoch 15/30\n",
      "286/286 [==============================] - 150s 523ms/step - loss: 5.7493 - accuracy: 0.6875 - val_loss: 3.5885 - val_accuracy: 0.7461 - lr: 0.0500\n",
      "Epoch 16/30\n",
      "286/286 [==============================] - 149s 520ms/step - loss: 5.7491 - accuracy: 0.6853 - val_loss: 3.3254 - val_accuracy: 0.7583 - lr: 0.0500\n",
      "Epoch 17/30\n",
      "286/286 [==============================] - 149s 520ms/step - loss: 5.6632 - accuracy: 0.6818 - val_loss: 3.2431 - val_accuracy: 0.7478 - lr: 0.0500\n",
      "Epoch 18/30\n",
      "286/286 [==============================] - 148s 517ms/step - loss: 5.4086 - accuracy: 0.6805 - val_loss: 3.3387 - val_accuracy: 0.7723 - lr: 0.0500\n",
      "Epoch 19/30\n",
      "286/286 [==============================] - 150s 524ms/step - loss: 5.2380 - accuracy: 0.6921 - val_loss: 3.7247 - val_accuracy: 0.7356 - lr: 0.0500\n",
      "Epoch 20/30\n",
      "286/286 [==============================] - 147s 514ms/step - loss: 4.6371 - accuracy: 0.7031 - val_loss: 2.8537 - val_accuracy: 0.7776 - lr: 0.0250\n",
      "Epoch 21/30\n",
      "286/286 [==============================] - 148s 518ms/step - loss: 4.0303 - accuracy: 0.7132 - val_loss: 2.3726 - val_accuracy: 0.7863 - lr: 0.0250\n",
      "Epoch 22/30\n",
      "286/286 [==============================] - 148s 517ms/step - loss: 3.8491 - accuracy: 0.7036 - val_loss: 2.4733 - val_accuracy: 0.7758 - lr: 0.0250\n",
      "Epoch 23/30\n",
      "286/286 [==============================] - 157s 551ms/step - loss: 3.6135 - accuracy: 0.7083 - val_loss: 2.5616 - val_accuracy: 0.7496 - lr: 0.0250\n",
      "Epoch 24/30\n",
      "286/286 [==============================] - 157s 548ms/step - loss: 3.4296 - accuracy: 0.7094 - val_loss: 2.3724 - val_accuracy: 0.7636 - lr: 0.0250\n",
      "Epoch 25/30\n",
      "286/286 [==============================] - 164s 574ms/step - loss: 3.4400 - accuracy: 0.6931 - val_loss: 1.7941 - val_accuracy: 0.7968 - lr: 0.0250\n",
      "Epoch 26/30\n",
      "286/286 [==============================] - 161s 564ms/step - loss: 3.1804 - accuracy: 0.7031 - val_loss: 1.9588 - val_accuracy: 0.7601 - lr: 0.0250\n",
      "Epoch 27/30\n",
      "286/286 [==============================] - 163s 567ms/step - loss: 3.0385 - accuracy: 0.7013 - val_loss: 1.8447 - val_accuracy: 0.7723 - lr: 0.0250\n",
      "Epoch 28/30\n",
      "286/286 [==============================] - 154s 539ms/step - loss: 3.1254 - accuracy: 0.6943 - val_loss: 1.9827 - val_accuracy: 0.7601 - lr: 0.0250\n",
      "Epoch 29/30\n",
      "286/286 [==============================] - 148s 516ms/step - loss: 2.9374 - accuracy: 0.7019 - val_loss: 2.2487 - val_accuracy: 0.7303 - lr: 0.0250\n",
      "Epoch 30/30\n",
      "286/286 [==============================] - 148s 518ms/step - loss: 2.6727 - accuracy: 0.7063 - val_loss: 1.5167 - val_accuracy: 0.8039 - lr: 0.0125\n"
     ]
    }
   ],
   "source": [
    "# ResNet50V2 모델 수정 및 완전 연결층 추가\n",
    "def create_teacher_model():\n",
    "    input = layers.Input(shape=(224, 224, 3))\n",
    "    base_model = ResNet50V2(input_tensor=input, include_top=False, weights='imagenet')\n",
    "    bm_output = base_model.output\n",
    "\n",
    "    # 모델 정규화 및 완전 연결층 추가\n",
    "    x = layers.GlobalAveragePooling2D()(bm_output)\n",
    "    x = layers.Dense(1024)(x)\n",
    "    x = layers.BatchNormalization()(x)\n",
    "    x = layers.Activation('relu')(x)\n",
    "    x = layers.Dropout(0.5)(x)\n",
    "    x = layers.Dense(7, activation=\"softmax\")(x)\n",
    "    \n",
    "    fine_tuned_model = Model(input, x)\n",
    "    return fine_tuned_model\n",
    "\n",
    "# 학습률 스케줄러\n",
    "def step_decay(epoch):\n",
    "    initial_lrate = 0.1\n",
    "    drop = 0.5\n",
    "    epochs_drop = 10.0\n",
    "    lrate = initial_lrate * math.pow(drop, math.floor((1+epoch)/epochs_drop))\n",
    "    return lrate\n",
    "\n",
    "# 이미지 파일 경로 및 라벨 정보를 담은 데이터 프레임 생성\n",
    "def create_dataframe(data_dir, exts):\n",
    "    data = []\n",
    "    for subdir, dirs, files in os.walk(data_dir):\n",
    "        for file in files:\n",
    "            ext = os.path.splitext(file)[-1].lower()\n",
    "            if ext in exts:\n",
    "                file_path = os.path.join(subdir, file)\n",
    "                try:\n",
    "                    with Image.open(file_path) as im:\n",
    "                        im.verify()  # 이미지 파일 유효성 검사\n",
    "                except (IOError, ValueError, UnidentifiedImageError):\n",
    "                    print(f\"Invalid image file '{file_path}', skipped.\")\n",
    "                    os.remove(file_path)\n",
    "                    continue\n",
    "                label = os.path.split(subdir)[-1].lower()\n",
    "                data.append((file_path, label))\n",
    "    return pd.DataFrame(data, columns=['filepath', 'label'])\n",
    "image_files = []\n",
    "labels = []\n",
    "\n",
    "for root, dirs, files in os.walk(data_dir):\n",
    "    for file in files:\n",
    "        if file.endswith('.jpg') or file.endswith('.jpeg') or file.endswith('.png'):\n",
    "            # 이미지 파일 경로와 라벨 정보 저장\n",
    "            img_path = os.path.join(root, file)\n",
    "            label = os.path.basename(os.path.dirname(img_path))\n",
    "            image_files.append(img_path)\n",
    "            labels.append(label)\n",
    "\n",
    "# 데이터 프레임 생성\n",
    "data = create_dataframe(data_dir, image_exts)\n",
    "\n",
    "# 학습 데이터 증가(ImageDataGenerator)\n",
    "train_datagen = ImageDataGenerator(rescale=1./255,\n",
    "                                   rotation_range=15,\n",
    "                                   shear_range=0.2,\n",
    "                                   zoom_range=0.2,\n",
    "                                   horizontal_flip=True,\n",
    "                                   fill_mode='nearest',\n",
    "                                   validation_split=0.2)\n",
    "train_data, valid_data = train_test_split(data, test_size=0.2, random_state=42, stratify=data['label'])\n",
    "# 훈련 및 검증 데이터 생성\n",
    "train_generator = train_datagen.flow_from_dataframe(dataframe=train_data,\n",
    "                                                    x_col='filepath',\n",
    "                                                    y_col='label',\n",
    "                                                    target_size=(224, 224),\n",
    "                                                    batch_size=32,\n",
    "                                                    class_mode='categorical',\n",
    "                                                    subset='training')\n",
    "\n",
    "validation_generator = train_datagen.flow_from_dataframe(dataframe=valid_data,\n",
    "                                                         x_col='filepath',\n",
    "                                                         y_col='label',\n",
    "                                                         target_size=(224, 224),\n",
    "                                                         batch_size=32,\n",
    "                                                         class_mode='categorical',\n",
    "                                                         subset='validation')\n",
    "\n",
    "# 모델 생성\n",
    "teacher_model = create_teacher_model()\n",
    "teacher_model.summary()\n",
    "\n",
    "# 기존 ResNet50V2 층 동결\n",
    "for layer in teacher_model.layers[:-1]:\n",
    "    layer.trainable = False\n",
    "\n",
    "# 최적화 알고리즘 변경(SGD) 및 학습률 스케줄러 콜백 추가\n",
    "opt = SGD(lr=0.1, decay=0.0, momentum=0.9, nesterov=False)\n",
    "lrate = LearningRateScheduler(step_decay)\n",
    "early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)\n",
    "callbacks_list = [lrate, early_stopping]\n",
    "teacher_model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])\n",
    "\n",
    "# 새로운 완전 연결층(Dense layer) 학습\n",
    "history = teacher_model.fit(train_generator, validation_data=validation_generator, epochs=30, verbose=1, callbacks=callbacks_list)\n",
    "\n",
    "# 동결을 풀기 전 학습률을 낮추어 미세조정을 효과적으로 수행\n",
    "opt = SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=False)\n",
    "teacher_model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])\n",
    "\n",
    "# 기존 ResNet50V2 층의 동결 해제\n",
    "for layer in teacher_model.layers[-20:]:\n",
    "    layer.trainable = True\n",
    "\n",
    "# 학습률 스케줄러 콜백 추가\n",
    "lrate = LearningRateScheduler(step_decay)\n",
    "callbacks_list = [lrate, early_stopping]\n",
    "\n",
    "# 미세 조정(fine-tuning) 실행\n",
    "history = teacher_model.fit(train_generator, validation_data=validation_generator, epochs=30, verbose=1, callbacks=callbacks_list)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "student model None\n"
     ]
    }
   ],
   "source": [
    "#Student 모델 생성 (기존 코드의 create_fine_tune_model 함수 이용)\n",
    "def create_student_model():\n",
    "    input_layer = Input(shape=(224, 224, 3))\n",
    "\n",
    "    x = layers.Conv2D(16, (3 ,3), activation='relu')(input_layer)\n",
    "    x = layers.MaxPooling2D((2 ,2))(x)\n",
    "\n",
    "    x = layers.Conv2D(32 , (3 ,3), activation='relu')(x)\n",
    "    x = layers.MaxPooling2D((2 ,2))(x)\n",
    "\n",
    "    x = layers.Flatten()(x)\n",
    "\n",
    "    x = layers.Dense(64, activation='relu')(x)\n",
    "\n",
    "    output_layer = layers.Dense(7, activation='softmax')(x)  # 분류 클래스 개수에 맞게 출력 레이어를 설정합니다.\n",
    "\n",
    "    model = Model(inputs=input_layer , outputs=output_layer)\n",
    "        \n",
    "    return model\n",
    "#student 모델 생성\n",
    "student_model = create_student_model()\n",
    "print(\"student model\", student_model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "ename": "StopIteration",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mStopIteration\u001b[0m                             Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[91], line 51\u001b[0m\n\u001b[0;32m     49\u001b[0m distiller\u001b[39m.\u001b[39mcompile(optimizer, metrics\u001b[39m=\u001b[39m[tf\u001b[39m.\u001b[39mkeras\u001b[39m.\u001b[39mmetrics\u001b[39m.\u001b[39mSparseCategoricalAccuracy()], distillation_loss_fn\u001b[39m=\u001b[39mtf\u001b[39m.\u001b[39mkeras\u001b[39m.\u001b[39mlosses\u001b[39m.\u001b[39mKLDivergence(), temperature\u001b[39m=\u001b[39m\u001b[39m3\u001b[39m)\n\u001b[0;32m     50\u001b[0m batch_size \u001b[39m=\u001b[39m \u001b[39m32\u001b[39m\n\u001b[1;32m---> 51\u001b[0m history \u001b[39m=\u001b[39m distiller\u001b[39m.\u001b[39;49mfit(train_generator, steps_per_epoch\u001b[39m=\u001b[39;49m\u001b[39mlen\u001b[39;49m(train_data) \u001b[39m/\u001b[39;49m\u001b[39m/\u001b[39;49m batch_size, epochs\u001b[39m=\u001b[39;49m\u001b[39m30\u001b[39;49m, validation_data\u001b[39m=\u001b[39;49mvalidation_generator, validation_steps\u001b[39m=\u001b[39;49m\u001b[39mlen\u001b[39;49m(validation_generator) \u001b[39m/\u001b[39;49m\u001b[39m/\u001b[39;49m batch_size, verbose\u001b[39m=\u001b[39;49m\u001b[39m1\u001b[39;49m)\n\u001b[0;32m     53\u001b[0m student \u001b[39m=\u001b[39m distiller\u001b[39m.\u001b[39mstudent\n\u001b[0;32m     54\u001b[0m student\u001b[39m.\u001b[39mcompile(metrics\u001b[39m=\u001b[39m[\u001b[39m\"\u001b[39m\u001b[39maccuracy\u001b[39m\u001b[39m\"\u001b[39m])\n",
      "File \u001b[1;32mc:\\Users\\layen\\anaconda3\\lib\\site-packages\\keras\\utils\\traceback_utils.py:67\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     65\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m \u001b[39mas\u001b[39;00m e:  \u001b[39m# pylint: disable=broad-except\u001b[39;00m\n\u001b[0;32m     66\u001b[0m   filtered_tb \u001b[39m=\u001b[39m _process_traceback_frames(e\u001b[39m.\u001b[39m__traceback__)\n\u001b[1;32m---> 67\u001b[0m   \u001b[39mraise\u001b[39;00m e\u001b[39m.\u001b[39mwith_traceback(filtered_tb) \u001b[39mfrom\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[0;32m     68\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[0;32m     69\u001b[0m   \u001b[39mdel\u001b[39;00m filtered_tb\n",
      "File \u001b[1;32mc:\\Users\\layen\\anaconda3\\lib\\site-packages\\keras\\engine\\data_adapter.py:860\u001b[0m, in \u001b[0;36mGeneratorDataAdapter._peek_and_restore\u001b[1;34m(x)\u001b[0m\n\u001b[0;32m    858\u001b[0m \u001b[39m@staticmethod\u001b[39m\n\u001b[0;32m    859\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_peek_and_restore\u001b[39m(x):\n\u001b[1;32m--> 860\u001b[0m   peek \u001b[39m=\u001b[39m \u001b[39mnext\u001b[39;49m(x)\n\u001b[0;32m    861\u001b[0m   \u001b[39mreturn\u001b[39;00m peek, itertools\u001b[39m.\u001b[39mchain([peek], x)\n",
      "\u001b[1;31mStopIteration\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# 1. Define the teacher model (pre-trained or trained with larger network)\n",
    "teacher = teacher_model  # Replace with your teacher model\n",
    "\n",
    "# 2. Define the student model (smaller than the teacher)\n",
    "student = student_model  # Replace with your student model\n",
    "\n",
    "class Distiller(tf.keras.Model):\n",
    "    def __init__(self, student, teacher):\n",
    "        super().__init__()\n",
    "        self.teacher = teacher\n",
    "        self.student = student\n",
    "        self.loss_tracker = tf.keras.metrics.Mean(name=\"distillation loss\")  \n",
    "        \n",
    "    def compile(self, optimizer, metrics, distillation_loss_fn, temperature=3):\n",
    "        super().compile(optimizer=optimizer, metrics=metrics)\n",
    "        self.distillation_loss_fn = distillation_loss_fn\n",
    "        self.temperature = temperature\n",
    "        \n",
    "    def train_step(self, data):\n",
    "        tf.keras.utils.disable_interactive_logging()\n",
    "        x, _ = data\n",
    "        # Forward pass of teacher\n",
    "        teacher_predictions = self.teacher(x, training=False)\n",
    "        \n",
    "        with tf.GradientTape() as tape:\n",
    "            # Forward pass of student\n",
    "            student_predictions = self.student(x, training=True)\n",
    "            \n",
    "            # Compute loss\n",
    "            distillation_loss = self.distillation_loss_fn(\n",
    "                tf.nn.softmax(teacher_predictions, axis=1),\n",
    "                tf.nn.softmax(student_predictions, axis=1),\n",
    "            )\n",
    "            \n",
    "        # Compute gradients\n",
    "        trainable_vars = self.student.trainable_variables\n",
    "        gradients = tape.gradient(distillation_loss, trainable_vars)\n",
    "        \n",
    "        # Update weights\n",
    "        self.optimizer.apply_gradients(zip(gradients, trainable_vars))\n",
    "        \n",
    "        # Report progress\n",
    "        self.loss_tracker.update_state(distillation_loss)\n",
    "        tf.keras.utils.enable_interactive_logging()\n",
    "        return {\"distillation loss\": self.loss_tracker.result()}\n",
    "        \n",
    "optimizer = tf.keras.optimizers.Adam()\n",
    "distiller = Distiller(student=student, teacher=teacher)\n",
    "distiller.compile(optimizer, metrics=[tf.keras.metrics.SparseCategoricalAccuracy()], distillation_loss_fn=tf.keras.losses.KLDivergence(), temperature=3)\n",
    "batch_size = 32\n",
    "history = distiller.fit(train_generator, steps_per_epoch=len(train_data) // batch_size, epochs=30, validation_data=validation_generator, validation_steps=len(validation_generator) // batch_size, verbose=1)\n",
    "\n",
    "student = distiller.student\n",
    "student.compile(metrics=[\"accuracy\"])\n",
    "_, accuracy = student.evaluate(validation_generator)\n",
    "print(f\"Validation accuracy: {accuracy * 100:.2f}%\")\n",
    "# student 모델 저장\n",
    "student.save(\"model\\student_model.h5\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}