SourceCode_of_Baseline.py

# -*- coding: utf-8 -*-
"""baseline.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1Mo-e4fFfN7YpoMFBP06KkwM5OgzRozgY

# Primary Reguirements
"""

import os 
import sys
import random
import json
import zipfile
from zipfile import ZipFile
 
print("[INFO]: Imported primary library")

print("[INFO]: Extracting dataset from zip file ...")
file_name = '/content/drive/My Drive/kaggle/tomato_dataset1/tomato.zip'
 
with ZipFile(file_name, 'r') as zip:
  zip.extractall()
  print("[INFO]: Extracted dataset zip file")
  
 #Import tensorflow and keras library
import tensorflow as tf
import keras_preprocessing
from tensorflow.keras.preprocessing import image
import pickle
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import TensorBoard
from keras.models import Sequential
from keras.layers import Convolution2D,MaxPooling2D,Flatten,Dense,Dropout
from keras.layers.normalization import BatchNormalization
from keras.optimizers import Adam
import keras
from tensorflow.keras import regularizers
from tensorflow.keras import activations
#from keras.layers.normalization import LayerNormalization
#from tf.keras.layers import LayerNormalization
from keras import layers
from keras.optimizers import SGD
from keras.callbacks import CSVLogger


print("[INFO]: Tensorflow version{}".format(tf.__version__))
state_gpu = tf.test.gpu_device_name()
print("[INFO]: GPU usage{0}".format(state_gpu))

# ImageDataGenerator and get train data and validation data
TRAINING_DIR = '/content/New Plant Diseases Dataset(Augmented)/train/' 
VALIDATION_DIR = '/content/New Plant Diseases Dataset(Augmented)/valid/'

# this is the augmentation configuration we will use for training
train_gen = ImageDataGenerator(rescale = 1./255)
valid_gen = ImageDataGenerator(rescale = 1./255)

TARGET_SIZE = (227,227)
TRAIN_BATCH_SIZE = 128
VALID_BATCH_SIZE = 32
SEED = 42
 
#Data Iterator
train_data = train_gen.flow_from_directory(
TRAINING_DIR,
target_size = TARGET_SIZE,
class_mode = 'categorical',
color_mode = "rgb",
batch_size = TRAIN_BATCH_SIZE,
shuffle = True,
seed = SEED
)
 
valid_data = valid_gen.flow_from_directory(
VALIDATION_DIR,
target_size = TARGET_SIZE,
class_mode = 'categorical',
color_mode = "rgb",
batch_size = VALID_BATCH_SIZE
)

import matplotlib.pyplot as plt
import seaborn as sns
sns.set()

"""# Baseline"""

import os 
import sys
import random
import json
import zipfile
from zipfile import ZipFile
 
print("[INFO]: Imported primary library")

print("[INFO]: Extracting dataset from zip file ...")
file_name = '/content/drive/My Drive/kaggle/tomato_dataset1/tomato.zip'
 
with ZipFile(file_name, 'r') as zip:
  zip.extractall()
  print("[INFO]: Extracted dataset zip file")

#Import tensorflow and keras library
import tensorflow as tf
import keras_preprocessing
from tensorflow.keras.preprocessing import image
import pickle
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import TensorBoard
from keras.models import Sequential
from keras.layers import Convolution2D,MaxPooling2D,Flatten,Dense,Dropout
from keras.layers.normalization import BatchNormalization
from keras.optimizers import Adam
import keras
from tensorflow.keras import regularizers
from tensorflow.keras import activations
#from keras.layers.normalization import LayerNormalization
#from tf.keras.layers import LayerNormalization
from keras import layers
 
print("[INFO]: Tensorflow version{}".format(tf.__version__))
state_gpu = tf.test.gpu_device_name()
print("[INFO]: GPU usage{0}".format(state_gpu))

# ImageDataGenerator and get train data and validation data
TRAINING_DIR = '/content/New Plant Diseases Dataset(Augmented)/train/' 
VALIDATION_DIR = '/content/New Plant Diseases Dataset(Augmented)/valid/'

# this is the augmentation configuration we will use for training
train_gen = ImageDataGenerator(rescale = 1./255)
valid_gen = ImageDataGenerator(rescale = 1./255)

TARGET_SIZE = (227,227)
TRAIN_BATCH_SIZE = 128
VALID_BATCH_SIZE = 32
SEED = 42
 
#Data Iterator
train_data = train_gen.flow_from_directory(
TRAINING_DIR,
target_size = TARGET_SIZE,
class_mode = 'categorical',
color_mode = "rgb",
batch_size = TRAIN_BATCH_SIZE,
shuffle = True,
seed = SEED
)
 
valid_data = valid_gen.flow_from_directory(
VALIDATION_DIR,
target_size = TARGET_SIZE,
class_mode = 'categorical',
color_mode = "rgb",
batch_size = VALID_BATCH_SIZE
)

#Show class indis and class name
for cl_indis, cl_name in enumerate(train_data.class_indices):
     print(cl_indis, cl_name)

# Base Model
 
# Initializing the CNN based AlexNet
model = Sequential()
 
#valid:zero padding, same:keep same dimensionality by add padding
 
# Convolution Step 1
model.add(Convolution2D(96, 11, strides = (4, 4), padding = 'valid', input_shape=(227, 227, 3), activation = 'relu'))
 
# Max Pooling Step 1
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))
 
# Convolution Step 2
model.add(Convolution2D(256, 5, strides = (1, 1), padding='same', activation = 'relu'))
 
# Max Pooling Step 2
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding='valid'))
 
 
# Convolution Step 3
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', activation = 'relu'))
 
# Convolution Step 4
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', activation = 'relu'))
 
 
# Convolution Step 5
model.add(Convolution2D(256, 3, strides=(1,1), padding='same', activation = 'relu'))
 
# Max Pooling Step 3
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))
 
# Flattening Step --> 6*6*256 = 9216
model.add(Flatten())
 
# Full Connection Steps
# 1st Fully Connected Layer
model.add(Dense(units = 4096, activation = 'relu'))
 
# 2nd Fully Connected Layer
model.add(Dense(units = 4096, activation = 'relu'))
 
# 3rd Fully Connected Layer
model.add(Dense(units = 10, activation = 'softmax'))
 
 
print("[INFO]: Model Summary")
 
model.summary()

from keras.optimizers import SGD
 
#LEARNING_RATE = 0.0001
LEARNING_RATE = 0.001
MOMENTUM = 0.9
 
# compile model
opt = SGD(lr=LEARNING_RATE, momentum=MOMENTUM)
model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])

from keras.callbacks import CSVLogger
 
#Log callback
filepath="baseline.log"
path = F"{filepath}" 
csv_logger = CSVLogger(path, separator=',', append=False)

TRAINING_NUM = train_data.samples #or train_data.n
VALID_NUM = valid_data.samples
EPOCHS = 25

STEP_SIZE_TRAIN = TRAINING_NUM // TRAIN_BATCH_SIZE 
STEP_SIZE_VALID = VALID_NUM // VALID_BATCH_SIZE

#Train Model
history = model.fit_generator(generator = train_data,
                    steps_per_epoch = STEP_SIZE_TRAIN,
                    validation_data = valid_data,
                    validation_steps = STEP_SIZE_VALID,
                    callbacks=[csv_logger],
                    epochs = EPOCHS
)

best_val_acc = max(history.history['val_accuracy'])
print("[INFO] Best Validation Accuracy: %",best_val_acc*100)

import matplotlib.pyplot as plt
import seaborn as sns
sns.set()

#Plot Accuracy and Loss

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)

#accuracy plot
plt.plot(epochs, acc, color='green', label='Training Accuracy')
plt.plot(epochs, val_acc, color='blue', label='Validation Accuracy')
plt.title('Training and Validation Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend()

plt.figure()

#loss plot
plt.plot(epochs, loss, color='pink', label='Training Loss')
plt.plot(epochs, val_loss, color='red', label='Validation Loss')
plt.title('Training and Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.show()

"""# BASELINE MODEL - He-Uniform Initilizer"""

# Initializing the CNN based AlexNet
model = Sequential()

#valid:zero padding, same:keep same dimensionality by add padding

# Convolution Step 1
model.add(Convolution2D(96, 11, strides = (4, 4), padding = 'valid', input_shape=(227, 227, 3), activation = 'relu', kernel_initializer='he_uniform'))

# Max Pooling Step 1
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))

# Convolution Step 2
model.add(Convolution2D(256, 5, strides = (1, 1), padding='same', activation = 'relu', kernel_initializer='he_uniform'))

# Max Pooling Step 2
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding='valid'))


# Convolution Step 3
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', activation = 'relu', kernel_initializer='he_uniform'))

# Convolution Step 4
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', activation = 'relu', kernel_initializer='he_uniform'))


# Convolution Step 5
model.add(Convolution2D(256, 3, strides=(1,1), padding='same', activation = 'relu', kernel_initializer='he_uniform'))

# Max Pooling Step 3
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))

# Flattening Step --> 6*6*256 = 9216
model.add(Flatten())

# Full Connection Steps
# 1st Fully Connected Layer
model.add(Dense(units = 4096, activation = 'relu', kernel_initializer='he_uniform'))

# 2nd Fully Connected Layer
model.add(Dense(units = 4096, activation = 'relu', kernel_initializer='he_uniform'))

# 3td Fully Connected Layer
model.add(Dense(units = 10, activation = 'softmax'))


print("[INFO]: Model Summary")

model.summary()

#LEARNING_RATE = 0.0001
LEARNING_RATE = 0.001
MOMENTUM = 0.9

# compile model
opt = SGD(lr=LEARNING_RATE, momentum=MOMENTUM)
model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])

#Log callback
filepath="he-uniform.log"
path = F"{filepath}" 
csv_logger = CSVLogger(path, separator=',', append=False)

TRAINING_NUM = train_data.samples #or train_data.n
VALID_NUM = valid_data.samples
EPOCHS = 25

STEP_SIZE_TRAIN = TRAINING_NUM // TRAIN_BATCH_SIZE 
STEP_SIZE_VALID = VALID_NUM // VALID_BATCH_SIZE

#Train Model
history = model.fit_generator(generator = train_data,
                    steps_per_epoch = STEP_SIZE_TRAIN,
                    validation_data = valid_data,
                    validation_steps = STEP_SIZE_VALID,
                    callbacks=[csv_logger],
                    epochs = EPOCHS
)

best_val_acc = max(history.history['val_accuracy'])
print("[INFO] Best Validation Accuracy: %",best_val_acc*100)

import matplotlib.pyplot as plt
import seaborn as sns
sns.set()

#Plot Accuracy and Loss

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)

#accuracy plot
plt.plot(epochs, acc, color='green', label='Training Accuracy')
plt.plot(epochs, val_acc, color='blue', label='Validation Accuracy')
plt.title('Training and Validation Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend()

plt.figure()

#loss plot
plt.plot(epochs, loss, color='pink', label='Training Loss')
plt.plot(epochs, val_loss, color='red', label='Validation Loss')
plt.title('Training and Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.show()

"""# BASELINE MODEL + He uniform + 1000 dense"""

# Initializing the CNN based AlexNet
model = Sequential()

#valid:zero padding, same:keep same dimensionality by add padding

# Convolution Step 1
model.add(Convolution2D(96, 11, strides = (4, 4), padding = 'valid', input_shape=(227, 227, 3), activation = 'relu', kernel_initializer='he_uniform'))

# Max Pooling Step 1
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))

# Convolution Step 2
model.add(Convolution2D(256, 5, strides = (1, 1), padding='same', activation = 'relu', kernel_initializer='he_uniform'))

# Max Pooling Step 2
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding='valid'))


# Convolution Step 3
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', activation = 'relu', kernel_initializer='he_uniform'))

# Convolution Step 4
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', activation = 'relu', kernel_initializer='he_uniform'))


# Convolution Step 5
model.add(Convolution2D(256, 3, strides=(1,1), padding='same', activation = 'relu', kernel_initializer='he_uniform'))

# Max Pooling Step 3
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))

# Flattening Step --> 6*6*256 = 9216
model.add(Flatten())

# Full Connection Steps
# 1st Fully Connected Layer
model.add(Dense(units = 4096, activation = 'relu', kernel_initializer='he_uniform'))

# 2nd Fully Connected Layer
model.add(Dense(units = 4096, activation = 'relu', kernel_initializer='he_uniform'))

# 3rd Fully Connected Layer
model.add(Dense(units = 1000, activation = 'relu', kernel_initializer='he_uniform'))

# 4td Fully Connected Layer
model.add(Dense(units = 10, activation = 'softmax'))


print("[INFO]: Model Summary")

model.summary()

#LEARNING_RATE = 0.0001
LEARNING_RATE = 0.001
MOMENTUM = 0.9

# compile model
opt = SGD(lr=LEARNING_RATE, momentum=MOMENTUM)
model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])

#Log callback
filepath="1000dense.log"
path = F"{filepath}" 
csv_logger = CSVLogger(path, separator=',', append=False)

TRAINING_NUM = train_data.samples #or train_data.n
VALID_NUM = valid_data.samples
EPOCHS = 25

STEP_SIZE_TRAIN = TRAINING_NUM // TRAIN_BATCH_SIZE 
STEP_SIZE_VALID = VALID_NUM // VALID_BATCH_SIZE

#Train Model
history = model.fit_generator(generator = train_data,
                    steps_per_epoch = STEP_SIZE_TRAIN,
                    validation_data = valid_data,
                    validation_steps = STEP_SIZE_VALID,
                    callbacks=[csv_logger],
                    epochs = EPOCHS
)

best_val_acc = max(history.history['val_accuracy'])
print("[INFO] Best Validation Accuracy: %",best_val_acc*100)

import matplotlib.pyplot as plt
import seaborn as sns
sns.set()

#Plot Accuracy and Loss

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)

#accuracy plot
plt.plot(epochs, acc, color='green', label='Training Accuracy')
plt.plot(epochs, val_acc, color='blue', label='Validation Accuracy')
plt.title('Training and Validation Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend()

plt.figure()

#loss plot
plt.plot(epochs, loss, color='pink', label='Training Loss')
plt.plot(epochs, val_loss, color='red', label='Validation Loss')
plt.title('Training and Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.show()

"""# BASELINE MODEL + He uniform + 1000 dense + Decide Opt"""

# Initializing the CNN based AlexNet
model = Sequential()

#valid:zero padding, same:keep same dimensionality by add padding

# Convolution Step 1
model.add(Convolution2D(96, 11, strides = (4, 4), padding = 'valid', input_shape=(227, 227, 3), activation = 'relu', kernel_initializer='he_uniform'))

# Max Pooling Step 1
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))

# Convolution Step 2
model.add(Convolution2D(256, 5, strides = (1, 1), padding='same', activation = 'relu', kernel_initializer='he_uniform'))

# Max Pooling Step 2
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding='valid'))


# Convolution Step 3
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', activation = 'relu', kernel_initializer='he_uniform'))

# Convolution Step 4
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', activation = 'relu', kernel_initializer='he_uniform'))


# Convolution Step 5
model.add(Convolution2D(256, 3, strides=(1,1), padding='same', activation = 'relu', kernel_initializer='he_uniform'))

# Max Pooling Step 3
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))

# Flattening Step --> 6*6*256 = 9216
model.add(Flatten())

# Full Connection Steps
# 1st Fully Connected Layer
model.add(Dense(units = 4096, activation = 'relu', kernel_initializer='he_uniform'))

# 2nd Fully Connected Layer
model.add(Dense(units = 4096, activation = 'relu', kernel_initializer='he_uniform'))

# 3rd Fully Connected Layer
model.add(Dense(units = 1000, activation = 'relu', kernel_initializer='he_uniform'))

# 4td Fully Connected Layer
model.add(Dense(units = 10, activation = 'softmax'))


print("[INFO]: Model Summary")

model.summary()

LEARNING_RATE = 0.001
MOMENTUM = 0.9

from keras.optimizers import Adam
import keras
opt = Adam(lr=LEARNING_RATE)
model.compile(optimizer=opt, loss=keras.losses.categorical_crossentropy, metrics=['accuracy'])

#Log callback
filepath="1000dense.log"
path = F"{filepath}" 
csv_logger = CSVLogger(path, separator=',', append=False)

TRAINING_NUM = train_data.samples #or train_data.n
VALID_NUM = valid_data.samples
EPOCHS = 25

STEP_SIZE_TRAIN = TRAINING_NUM // TRAIN_BATCH_SIZE 
STEP_SIZE_VALID = VALID_NUM // VALID_BATCH_SIZE

#Train Model
history = model.fit_generator(generator = train_data,
                    steps_per_epoch = STEP_SIZE_TRAIN,
                    validation_data = valid_data,
                    validation_steps = STEP_SIZE_VALID,
                    callbacks=[csv_logger],
                    epochs = EPOCHS
)

best_val_acc = max(history.history['val_accuracy'])
print("[INFO] Best Validation Accuracy: %",best_val_acc*100)

import matplotlib.pyplot as plt
import seaborn as sns
sns.set()

#Plot Accuracy and Loss

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)

#accuracy plot
plt.plot(epochs, acc, color='green', label='Training Accuracy')
plt.plot(epochs, val_acc, color='blue', label='Validation Accuracy')
plt.title('Training and Validation Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend()

plt.figure()

#loss plot
plt.plot(epochs, loss, color='pink', label='Training Loss')
plt.plot(epochs, val_loss, color='red', label='Validation Loss')
plt.title('Training and Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.show()

"""# Decide learning rate"""

# Initializing the CNN based AlexNet
model = Sequential()

#valid:zero padding, same:keep same dimensionality by add padding

# Convolution Step 1
model.add(Convolution2D(96, 11, strides = (4, 4), padding = 'valid', input_shape=(227, 227, 3), activation = 'relu', kernel_initializer='he_uniform'))

# Max Pooling Step 1
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))

# Convolution Step 2
model.add(Convolution2D(256, 5, strides = (1, 1), padding='same', activation = 'relu', kernel_initializer='he_uniform'))

# Max Pooling Step 2
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding='valid'))


# Convolution Step 3
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', activation = 'relu', kernel_initializer='he_uniform'))

# Convolution Step 4
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', activation = 'relu', kernel_initializer='he_uniform'))


# Convolution Step 5
model.add(Convolution2D(256, 3, strides=(1,1), padding='same', activation = 'relu', kernel_initializer='he_uniform'))

# Max Pooling Step 3
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))

# Flattening Step --> 6*6*256 = 9216
model.add(Flatten())

# Full Connection Steps
# 1st Fully Connected Layer
model.add(Dense(units = 4096, activation = 'relu', kernel_initializer='he_uniform'))

# 2nd Fully Connected Layer
model.add(Dense(units = 4096, activation = 'relu', kernel_initializer='he_uniform'))

# 3rd Fully Connected Layer
model.add(Dense(units = 1000, activation = 'relu', kernel_initializer='he_uniform'))

# 4td Fully Connected Layer
model.add(Dense(units = 10, activation = 'softmax'))


print("[INFO]: Model Summary")

model.summary()

#LEARNING_RATE = 0.0001
LEARNING_RATE = 0.01
MOMENTUM = 0.9

# compile model
opt = SGD(lr=LEARNING_RATE, momentum=MOMENTUM)
model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])

#Log callback
filepath="lr.log"
path = F"{filepath}" 
csv_logger = CSVLogger(path, separator=',', append=False)

TRAINING_NUM = train_data.samples #or train_data.n
VALID_NUM = valid_data.samples
EPOCHS = 25

STEP_SIZE_TRAIN = TRAINING_NUM // TRAIN_BATCH_SIZE 
STEP_SIZE_VALID = VALID_NUM // VALID_BATCH_SIZE

#Train Model
history = model.fit_generator(generator = train_data,
                    steps_per_epoch = STEP_SIZE_TRAIN,
                    validation_data = valid_data,
                    validation_steps = STEP_SIZE_VALID,
                    callbacks=[csv_logger],
                    epochs = EPOCHS
)

best_val_acc = max(history.history['val_accuracy'])
print("[INFO] Best Validation Accuracy: %",best_val_acc*100)

import matplotlib.pyplot as plt
import seaborn as sns
sns.set()

#Plot Accuracy and Loss

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)

#accuracy plot
plt.plot(epochs, acc, color='green', label='Training Accuracy')
plt.plot(epochs, val_acc, color='blue', label='Validation Accuracy')
plt.title('Training and Validation Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend()

plt.figure()

#loss plot
plt.plot(epochs, loss, color='pink', label='Training Loss')
plt.plot(epochs, val_loss, color='red', label='Validation Loss')
plt.title('Training and Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.show()

"""# Data Augmentation"""

# this is the augmentation configuration we will use for training
train_gen = ImageDataGenerator(
rescale = 1./255,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True,
fill_mode='nearest')

#train_gen = ImageDataGenerator(rescale = 1./255)
valid_gen = ImageDataGenerator(rescale = 1./255)

TARGET_SIZE = (227,227)
TRAIN_BATCH_SIZE = 128
VALID_BATCH_SIZE = 32
SEED = 42

#Data Iterator
train_data = train_gen.flow_from_directory(
TRAINING_DIR,
target_size = TARGET_SIZE,
class_mode = 'categorical',
color_mode = "rgb",
batch_size = TRAIN_BATCH_SIZE,
shuffle = True,
seed = SEED
)

valid_data = valid_gen.flow_from_directory(
VALIDATION_DIR,
target_size = TARGET_SIZE,
class_mode = 'categorical',
color_mode = "rgb",
batch_size = VALID_BATCH_SIZE
)

# Initializing the CNN based AlexNet
model = Sequential()

#valid:zero padding, same:keep same dimensionality by add padding

# Convolution Step 1
model.add(Convolution2D(96, 11, strides = (4, 4), padding = 'valid', input_shape=(227, 227, 3), activation = 'relu', kernel_initializer='he_uniform'))

# Max Pooling Step 1
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))

# Convolution Step 2
model.add(Convolution2D(256, 5, strides = (1, 1), padding='same', activation = 'relu', kernel_initializer='he_uniform'))

# Max Pooling Step 2
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding='valid'))


# Convolution Step 3
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', activation = 'relu', kernel_initializer='he_uniform'))

# Convolution Step 4
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', activation = 'relu', kernel_initializer='he_uniform'))


# Convolution Step 5
model.add(Convolution2D(256, 3, strides=(1,1), padding='same', activation = 'relu', kernel_initializer='he_uniform'))

# Max Pooling Step 3
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))

# Flattening Step --> 6*6*256 = 9216
model.add(Flatten())

# Full Connection Steps
# 1st Fully Connected Layer
model.add(Dense(units = 4096, activation = 'relu', kernel_initializer='he_uniform'))

# 2nd Fully Connected Layer
model.add(Dense(units = 4096, activation = 'relu', kernel_initializer='he_uniform'))

# 3rd Fully Connected Layer
model.add(Dense(units = 1000, activation = 'relu', kernel_initializer='he_uniform'))

# 4td Fully Connected Layer
model.add(Dense(units = 10, activation = 'softmax'))


print("[INFO]: Model Summary")

model.summary()

#LEARNING_RATE = 0.0001
LEARNING_RATE = 0.001
MOMENTUM = 0.9

# compile model
opt = SGD(lr=LEARNING_RATE, momentum=MOMENTUM)
model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])

#Log callback
filepath="data_aug.log"
path = F"{filepath}" 
csv_logger = CSVLogger(path, separator=',', append=False)

TRAINING_NUM = train_data.samples #or train_data.n
VALID_NUM = valid_data.samples
EPOCHS = 25

STEP_SIZE_TRAIN = TRAINING_NUM // TRAIN_BATCH_SIZE 
STEP_SIZE_VALID = VALID_NUM // VALID_BATCH_SIZE

#Train Model
history = model.fit_generator(generator = train_data,
                    steps_per_epoch = STEP_SIZE_TRAIN,
                    validation_data = valid_data,
                    validation_steps = STEP_SIZE_VALID,
                    callbacks=[csv_logger],
                    epochs = EPOCHS
)

best_val_acc = max(history.history['val_accuracy'])
print("[INFO] Best Validation Accuracy: %",best_val_acc*100)

#Plot Accuracy and Loss

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)

#accuracy plot
plt.plot(epochs, acc, color='green', label='Training Accuracy')
plt.plot(epochs, val_acc, color='blue', label='Validation Accuracy')
plt.title('Training and Validation Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend()

plt.figure()

#loss plot
plt.plot(epochs, loss, color='pink', label='Training Loss')
plt.plot(epochs, val_loss, color='red', label='Validation Loss')
plt.title('Training and Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.show()

"""# Dropout 0.5"""

# this is the augmentation configuration we will use for training

train_gen = ImageDataGenerator(rescale = 1./255)
valid_gen = ImageDataGenerator(rescale = 1./255)

TARGET_SIZE = (227,227)
TRAIN_BATCH_SIZE = 128
VALID_BATCH_SIZE = 32
SEED = 42

#Data Iterator
train_data = train_gen.flow_from_directory(
TRAINING_DIR,
target_size = TARGET_SIZE,
class_mode = 'categorical',
color_mode = "rgb",
batch_size = TRAIN_BATCH_SIZE,
shuffle = True,
seed = SEED
)

valid_data = valid_gen.flow_from_directory(
VALIDATION_DIR,
target_size = TARGET_SIZE,
class_mode = 'categorical',
color_mode = "rgb",
batch_size = VALID_BATCH_SIZE
)

from tensorflow.keras import regularizers
from tensorflow.keras import activations
#from keras.layers.normalization import LayerNormalization
#from tf.keras.layers import LayerNormalization
from keras import layers
# Initializing the CNN based AlexNet
model = Sequential()

#valid:zero padding, same:keep same dimensionality by add padding

# Convolution Step 1
model.add(Convolution2D(96, 11, strides = (4, 4), padding = 'valid', input_shape=(227, 227, 3), kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Max Pooling Step 1
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))


# Convolution Step 2
model.add(Convolution2D(256, 5, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Max Pooling Step 2
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding='valid'))


# Convolution Step 3
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))


# Convolution Step 4
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same',  kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))


# Convolution Step 5
model.add(Convolution2D(256, 3, strides=(1,1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Max Pooling Step 3
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))


# Flattening Step --> 6*6*256 = 9216
model.add(Flatten())

# Full Connection Steps
# 1st Fully Connected Layer
model.add(Dense(units = 4096,  kernel_initializer='he_uniform')) 
model.add(layers.Activation(activations.relu))
model.add(Dropout(0.5))

# 2nd Fully Connected Layer
model.add(Dense(units = 4096,kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
model.add(Dropout(0.5))

# 3rd Fully Connected Layer
model.add(Dense(units = 1000,  kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
model.add(Dropout(0.5))

# 4td Fully Connected Layer
model.add(Dense(units = 10, activation = 'softmax'))



print("[INFO]: Model Summary")

model.summary()

#LEARNING_RATE = 0.0001
LEARNING_RATE = 0.001
MOMENTUM = 0.9

# compile model
opt = SGD(lr=LEARNING_RATE, momentum=MOMENTUM)
model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])

#Log callback
filepath="dr05.log"
path = F"{filepath}" 
csv_logger = CSVLogger(path, separator=',', append=False)

TRAINING_NUM = train_data.samples #or train_data.n
VALID_NUM = valid_data.samples
EPOCHS = 25

STEP_SIZE_TRAIN = TRAINING_NUM // TRAIN_BATCH_SIZE 
STEP_SIZE_VALID = VALID_NUM // VALID_BATCH_SIZE

#Train Model
history = model.fit_generator(generator = train_data,
                    steps_per_epoch = STEP_SIZE_TRAIN,
                    validation_data = valid_data,
                    validation_steps = STEP_SIZE_VALID,
                    callbacks=[csv_logger],
                    epochs = EPOCHS
)

best_val_acc = max(history.history['val_accuracy'])
print("[INFO] Best Validation Accuracy: %",best_val_acc*100)

#Plot Accuracy and Loss

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)

#accuracy plot
plt.plot(epochs, acc, color='green', label='Training Accuracy')
plt.plot(epochs, val_acc, color='blue', label='Validation Accuracy')
plt.title('Training and Validation Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend()

plt.figure()

#loss plot
plt.plot(epochs, loss, color='pink', label='Training Loss')
plt.plot(epochs, val_loss, color='red', label='Validation Loss')
plt.title('Training and Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.show()

"""# FC layer Dropout 0.5 + Conv layer Dropout 0.2"""

from tensorflow.keras import regularizers
from tensorflow.keras import activations
#from keras.layers.normalization import LayerNormalization
#from tf.keras.layers import LayerNormalization
from keras import layers
# Initializing the CNN based AlexNet
model = Sequential()

#valid:zero padding, same:keep same dimensionality by add padding

# Convolution Step 1
model.add(Convolution2D(96, 11, strides = (4, 4), padding = 'valid', input_shape=(227, 227, 3), kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Max Pooling Step 1
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))
model.add(Dropout(0.2))

# Convolution Step 2
model.add(Convolution2D(256, 5, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Max Pooling Step 2
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding='valid'))
model.add(Dropout(0.2))

# Convolution Step 3
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
model.add(Dropout(0.2))

# Convolution Step 4
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same',  kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
model.add(Dropout(0.2))

# Convolution Step 5
model.add(Convolution2D(256, 3, strides=(1,1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Max Pooling Step 3
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))
model.add(Dropout(0.2))

# Flattening Step --> 6*6*256 = 9216
model.add(Flatten())

# Full Connection Steps
# 1st Fully Connected Layer
model.add(Dense(units = 4096,  kernel_initializer='he_uniform')) 
model.add(layers.Activation(activations.relu))
model.add(Dropout(0.5))

# 2nd Fully Connected Layer
model.add(Dense(units = 4096,kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
model.add(Dropout(0.5))

# 3rd Fully Connected Layer
model.add(Dense(units = 1000,  kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
model.add(Dropout(0.5))

# 4td Fully Connected Layer
model.add(Dense(units = 10, activation = 'softmax'))



print("[INFO]: Model Summary")

model.summary()

#LEARNING_RATE = 0.0001
LEARNING_RATE = 0.001
MOMENTUM = 0.9

# compile model
opt = SGD(lr=LEARNING_RATE, momentum=MOMENTUM)
model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])

#Log callback
filepath="dr05-dr02.log"
path = F"{filepath}" 
csv_logger = CSVLogger(path, separator=',', append=False)

TRAINING_NUM = train_data.samples #or train_data.n
VALID_NUM = valid_data.samples
EPOCHS = 25

STEP_SIZE_TRAIN = TRAINING_NUM // TRAIN_BATCH_SIZE 
STEP_SIZE_VALID = VALID_NUM // VALID_BATCH_SIZE

#Train Model
history = model.fit_generator(generator = train_data,
                    steps_per_epoch = STEP_SIZE_TRAIN,
                    validation_data = valid_data,
                    validation_steps = STEP_SIZE_VALID,
                    callbacks=[csv_logger],
                    epochs = EPOCHS
)

best_val_acc = max(history.history['val_accuracy'])
print("[INFO] Best Validation Accuracy: %",best_val_acc*100)

#Plot Accuracy and Loss

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)

#accuracy plot
plt.plot(epochs, acc, color='green', label='Training Accuracy')
plt.plot(epochs, val_acc, color='blue', label='Validation Accuracy')
plt.title('Training and Validation Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend()

plt.figure()

#loss plot
plt.plot(epochs, loss, color='pink', label='Training Loss')
plt.plot(epochs, val_loss, color='red', label='Validation Loss')
plt.title('Training and Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.show()

"""# Dropout 0.4"""

# Initializing the CNN based AlexNet
model = Sequential()

#valid:zero padding, same:keep same dimensionality by add padding

# Convolution Step 1
model.add(Convolution2D(96, 11, strides = (4, 4), padding = 'valid', input_shape=(227, 227, 3), kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Max Pooling Step 1
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))


# Convolution Step 2
model.add(Convolution2D(256, 5, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Max Pooling Step 2
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding='valid'))


# Convolution Step 3
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))


# Convolution Step 4
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same',  kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))


# Convolution Step 5
model.add(Convolution2D(256, 3, strides=(1,1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Max Pooling Step 3
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))


# Flattening Step --> 6*6*256 = 9216
model.add(Flatten())

# Full Connection Steps
# 1st Fully Connected Layer
model.add(Dense(units = 4096,  kernel_initializer='he_uniform')) 
model.add(layers.Activation(activations.relu))
model.add(Dropout(0.4))

# 2nd Fully Connected Layer
model.add(Dense(units = 4096,kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
model.add(Dropout(0.4))

# 3rd Fully Connected Layer
model.add(Dense(units = 1000,  kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
model.add(Dropout(0.4))

# 4td Fully Connected Layer
model.add(Dense(units = 10, activation = 'softmax'))



print("[INFO]: Model Summary")

model.summary()

#LEARNING_RATE = 0.0001
LEARNING_RATE = 0.001
MOMENTUM = 0.9

# compile model
opt = SGD(lr=LEARNING_RATE, momentum=MOMENTUM)
model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])

#Log callback
filepath="dr04.log"
path = F"{filepath}" 
csv_logger = CSVLogger(path, separator=',', append=False)

TRAINING_NUM = train_data.samples #or train_data.n
VALID_NUM = valid_data.samples
EPOCHS = 25

STEP_SIZE_TRAIN = TRAINING_NUM // TRAIN_BATCH_SIZE 
STEP_SIZE_VALID = VALID_NUM // VALID_BATCH_SIZE

#Train Model
history = model.fit_generator(generator = train_data,
                    steps_per_epoch = STEP_SIZE_TRAIN,
                    validation_data = valid_data,
                    validation_steps = STEP_SIZE_VALID,
                    callbacks=[csv_logger],
                    epochs = EPOCHS
)

best_val_acc = max(history.history['val_accuracy'])
print("[INFO] Best Validation Accuracy: %",best_val_acc*100)

#Plot Accuracy and Loss

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)

#accuracy plot
plt.plot(epochs, acc, color='green', label='Training Accuracy')
plt.plot(epochs, val_acc, color='blue', label='Validation Accuracy')
plt.title('Training and Validation Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend()

plt.figure()

#loss plot
plt.plot(epochs, loss, color='pink', label='Training Loss')
plt.plot(epochs, val_loss, color='red', label='Validation Loss')
plt.title('Training and Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.show()

"""# Dropout 0.2"""

from tensorflow.keras import regularizers
from tensorflow.keras import activations
#from keras.layers.normalization import LayerNormalization
#from tf.keras.layers import LayerNormalization
from keras import layers
# Initializing the CNN based AlexNet
model = Sequential()

#valid:zero padding, same:keep same dimensionality by add padding

# Convolution Step 1
model.add(Convolution2D(96, 11, strides = (4, 4), padding = 'valid', input_shape=(227, 227, 3), kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Max Pooling Step 1
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))


# Convolution Step 2
model.add(Convolution2D(256, 5, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Max Pooling Step 2
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding='valid'))


# Convolution Step 3
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))


# Convolution Step 4
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same',  kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))


# Convolution Step 5
model.add(Convolution2D(256, 3, strides=(1,1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Max Pooling Step 3
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))


# Flattening Step --> 6*6*256 = 9216
model.add(Flatten())

# Full Connection Steps
# 1st Fully Connected Layer
model.add(Dense(units = 4096,  kernel_initializer='he_uniform')) 
model.add(layers.Activation(activations.relu))
model.add(Dropout(0.2))

# 2nd Fully Connected Layer
model.add(Dense(units = 4096,kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
model.add(Dropout(0.2))

# 3rd Fully Connected Layer
model.add(Dense(units = 1000,  kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
model.add(Dropout(0.2))

# 4td Fully Connected Layer
model.add(Dense(units = 10, activation = 'softmax'))



print("[INFO]: Model Summary")

model.summary()

#LEARNING_RATE = 0.0001
LEARNING_RATE = 0.001
MOMENTUM = 0.9

# compile model
opt = SGD(lr=LEARNING_RATE, momentum=MOMENTUM)
model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])

#Log callback
filepath="dr02log"
path = F"{filepath}" 
csv_logger = CSVLogger(path, separator=',', append=False)

TRAINING_NUM = train_data.samples #or train_data.n
VALID_NUM = valid_data.samples
EPOCHS = 25

STEP_SIZE_TRAIN = TRAINING_NUM // TRAIN_BATCH_SIZE 
STEP_SIZE_VALID = VALID_NUM // VALID_BATCH_SIZE

#Train Model
history = model.fit_generator(generator = train_data,
                    steps_per_epoch = STEP_SIZE_TRAIN,
                    validation_data = valid_data,
                    validation_steps = STEP_SIZE_VALID,
                    callbacks=[csv_logger],
                    epochs = EPOCHS
)

best_val_acc = max(history.history['val_accuracy'])
print("[INFO] Best Validation Accuracy: %",best_val_acc*100)

#Plot Accuracy and Loss

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)

#accuracy plot
plt.plot(epochs, acc, color='green', label='Training Accuracy')
plt.plot(epochs, val_acc, color='blue', label='Validation Accuracy')
plt.title('Training and Validation Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend()

plt.figure()

#loss plot
plt.plot(epochs, loss, color='pink', label='Training Loss')
plt.plot(epochs, val_loss, color='red', label='Validation Loss')
plt.title('Training and Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.show()

"""# l2 reg 0.1"""

from tensorflow.keras import regularizers
from tensorflow.keras import activations
#from keras.layers.normalization import LayerNormalization
#from tf.keras.layers import LayerNormalization
from keras import layers
# Initializing the CNN based AlexNet
model = Sequential()

#valid:zero padding, same:keep same dimensionality by add padding

# Convolution Step 1
model.add(Convolution2D(96, 11, strides = (4, 4), padding = 'valid', input_shape=(227, 227, 3), kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Max Pooling Step 1
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))

# Convolution Step 2
model.add(Convolution2D(256, 5, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Max Pooling Step 2
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding='valid'))

# Convolution Step 3
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Convolution Step 4
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same',  kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Convolution Step 5
model.add(Convolution2D(256, 3, strides=(1,1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Max Pooling Step 3
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))


# Flattening Step --> 6*6*256 = 9216
model.add(Flatten())

# Full Connection Steps
# 1st Fully Connected Layer
model.add(Dense(units = 4096,  kernel_initializer='he_uniform',kernel_regularizer=regularizers.l2(0.1))) 
model.add(layers.Activation(activations.relu))

# 2nd Fully Connected Layer
model.add(Dense(units = 4096,kernel_initializer='he_uniform', kernel_regularizer=regularizers.l2(0.1)))
model.add(layers.Activation(activations.relu))


# 3rd Fully Connected Layer
model.add(Dense(units = 1000,  kernel_initializer='he_uniform',kernel_regularizer=regularizers.l2(0.1) ))
model.add(layers.Activation(activations.relu))


# 4td Fully Connected Layer
model.add(Dense(units = 10, activation = 'softmax'))



print("[INFO]: Model Summary")

model.summary()

#LEARNING_RATE = 0.0001
LEARNING_RATE = 0.001
MOMENTUM = 0.9

# compile model
opt = SGD(lr=LEARNING_RATE, momentum=MOMENTUM)
model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])

#Log callback
filepath="lr01.log"
path = F"{filepath}" 
csv_logger = CSVLogger(path, separator=',', append=False)

TRAINING_NUM = train_data.samples #or train_data.n
VALID_NUM = valid_data.samples
EPOCHS = 25

STEP_SIZE_TRAIN = TRAINING_NUM // TRAIN_BATCH_SIZE 
STEP_SIZE_VALID = VALID_NUM // VALID_BATCH_SIZE

#Train Model
history = model.fit_generator(generator = train_data,
                    steps_per_epoch = STEP_SIZE_TRAIN,
                    validation_data = valid_data,
                    validation_steps = STEP_SIZE_VALID,
                    callbacks=[csv_logger],
                    epochs = EPOCHS
)

best_val_acc = max(history.history['val_accuracy'])
print("[INFO] Best Validation Accuracy: %",best_val_acc*100)

#Plot Accuracy and Loss

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)

#accuracy plot
plt.plot(epochs, acc, color='green', label='Training Accuracy')
plt.plot(epochs, val_acc, color='blue', label='Validation Accuracy')
plt.title('Training and Validation Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend()

plt.figure()

#loss plot
plt.plot(epochs, loss, color='pink', label='Training Loss')
plt.plot(epochs, val_loss, color='red', label='Validation Loss')
plt.title('Training and Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.show()

"""# l2 reg 0.01"""

# Initializing the CNN based AlexNet
model = Sequential()

#valid:zero padding, same:keep same dimensionality by add padding

# Convolution Step 1
model.add(Convolution2D(96, 11, strides = (4, 4), padding = 'valid', input_shape=(227, 227, 3), kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Max Pooling Step 1
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))

# Convolution Step 2
model.add(Convolution2D(256, 5, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Max Pooling Step 2
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding='valid'))

# Convolution Step 3
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Convolution Step 4
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same',  kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Convolution Step 5
model.add(Convolution2D(256, 3, strides=(1,1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Max Pooling Step 3
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))


# Flattening Step --> 6*6*256 = 9216
model.add(Flatten())

# Full Connection Steps
# 1st Fully Connected Layer
model.add(Dense(units = 4096,  kernel_initializer='he_uniform',kernel_regularizer=regularizers.l2(0.01))) 
model.add(layers.Activation(activations.relu))

# 2nd Fully Connected Layer
model.add(Dense(units = 4096,kernel_initializer='he_uniform', kernel_regularizer=regularizers.l2(0.01)))
model.add(layers.Activation(activations.relu))


# 3rd Fully Connected Layer
model.add(Dense(units = 1000,  kernel_initializer='he_uniform',kernel_regularizer=regularizers.l2(0.01) ))
model.add(layers.Activation(activations.relu))


# 4td Fully Connected Layer
model.add(Dense(units = 10, activation = 'softmax'))



print("[INFO]: Model Summary")

model.summary()

#LEARNING_RATE = 0.0001
LEARNING_RATE = 0.001
MOMENTUM = 0.9

# compile model
opt = SGD(lr=LEARNING_RATE, momentum=MOMENTUM)
model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])

#Log callback
filepath="lr001.log"
path = F"{filepath}" 
csv_logger = CSVLogger(path, separator=',', append=False)

TRAINING_NUM = train_data.samples #or train_data.n
VALID_NUM = valid_data.samples
EPOCHS = 25

STEP_SIZE_TRAIN = TRAINING_NUM // TRAIN_BATCH_SIZE 
STEP_SIZE_VALID = VALID_NUM // VALID_BATCH_SIZE

#Train Model
history = model.fit_generator(generator = train_data,
                    steps_per_epoch = STEP_SIZE_TRAIN,
                    validation_data = valid_data,
                    validation_steps = STEP_SIZE_VALID,
                    callbacks=[csv_logger],
                    epochs = EPOCHS
)

best_val_acc = max(history.history['val_accuracy'])
print("[INFO] Best Validation Accuracy: %",best_val_acc*100)

#Plot Accuracy and Loss

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)

#accuracy plot
plt.plot(epochs, acc, color='green', label='Training Accuracy')
plt.plot(epochs, val_acc, color='blue', label='Validation Accuracy')
plt.title('Training and Validation Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend()

plt.figure()

#loss plot
plt.plot(epochs, loss, color='pink', label='Training Loss')
plt.plot(epochs, val_loss, color='red', label='Validation Loss')
plt.title('Training and Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.show()

"""# l2 0.05"""

# Initializing the CNN based AlexNet
model = Sequential()

#valid:zero padding, same:keep same dimensionality by add padding

# Convolution Step 1
model.add(Convolution2D(96, 11, strides = (4, 4), padding = 'valid', input_shape=(227, 227, 3), kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Max Pooling Step 1
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))

# Convolution Step 2
model.add(Convolution2D(256, 5, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Max Pooling Step 2
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding='valid'))

# Convolution Step 3
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Convolution Step 4
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same',  kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Convolution Step 5
model.add(Convolution2D(256, 3, strides=(1,1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Max Pooling Step 3
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))


# Flattening Step --> 6*6*256 = 9216
model.add(Flatten())

# Full Connection Steps
# 1st Fully Connected Layer
model.add(Dense(units = 4096,  kernel_initializer='he_uniform',kernel_regularizer=regularizers.l2(0.05))) 
model.add(layers.Activation(activations.relu))

# 2nd Fully Connected Layer
model.add(Dense(units = 4096,kernel_initializer='he_uniform', kernel_regularizer=regularizers.l2(0.05)))
model.add(layers.Activation(activations.relu))


# 3rd Fully Connected Layer
model.add(Dense(units = 1000,  kernel_initializer='he_uniform',kernel_regularizer=regularizers.l2(0.05) ))
model.add(layers.Activation(activations.relu))


# 4td Fully Connected Layer
model.add(Dense(units = 10, activation = 'softmax'))



print("[INFO]: Model Summary")

model.summary()

#LEARNING_RATE = 0.0001
LEARNING_RATE = 0.001
MOMENTUM = 0.9

# compile model
opt = SGD(lr=LEARNING_RATE, momentum=MOMENTUM)
model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])

#Log callback
filepath="lr005.log"
path = F"{filepath}" 
csv_logger = CSVLogger(path, separator=',', append=False)

TRAINING_NUM = train_data.samples #or train_data.n
VALID_NUM = valid_data.samples
EPOCHS = 25

STEP_SIZE_TRAIN = TRAINING_NUM // TRAIN_BATCH_SIZE 
STEP_SIZE_VALID = VALID_NUM // VALID_BATCH_SIZE

#Train Model
history = model.fit_generator(generator = train_data,
                    steps_per_epoch = STEP_SIZE_TRAIN,
                    validation_data = valid_data,
                    validation_steps = STEP_SIZE_VALID,
                    callbacks=[csv_logger],
                    epochs = EPOCHS
)

best_val_acc = max(history.history['val_accuracy'])
print("[INFO] Best Validation Accuracy: %",best_val_acc*100)

#Plot Accuracy and Loss

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)

#accuracy plot
plt.plot(epochs, acc, color='green', label='Training Accuracy')
plt.plot(epochs, val_acc, color='blue', label='Validation Accuracy')
plt.title('Training and Validation Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend()

plt.figure()

#loss plot
plt.plot(epochs, loss, color='pink', label='Training Loss')
plt.plot(epochs, val_loss, color='red', label='Validation Loss')
plt.title('Training and Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.show()

"""# l2 0.001"""

# Initializing the CNN based AlexNet
model = Sequential()

#valid:zero padding, same:keep same dimensionality by add padding

# Convolution Step 1
model.add(Convolution2D(96, 11, strides = (4, 4), padding = 'valid', input_shape=(227, 227, 3), kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Max Pooling Step 1
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))

# Convolution Step 2
model.add(Convolution2D(256, 5, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Max Pooling Step 2
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding='valid'))

# Convolution Step 3
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Convolution Step 4
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same',  kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Convolution Step 5
model.add(Convolution2D(256, 3, strides=(1,1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Max Pooling Step 3
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))


# Flattening Step --> 6*6*256 = 9216
model.add(Flatten())

# Full Connection Steps
# 1st Fully Connected Layer
model.add(Dense(units = 4096,  kernel_initializer='he_uniform',kernel_regularizer=regularizers.l2(0.001))) 
model.add(layers.Activation(activations.relu))

# 2nd Fully Connected Layer
model.add(Dense(units = 4096,kernel_initializer='he_uniform', kernel_regularizer=regularizers.l2(0.001)))
model.add(layers.Activation(activations.relu))


# 3rd Fully Connected Layer
model.add(Dense(units = 1000,  kernel_initializer='he_uniform',kernel_regularizer=regularizers.l2(0.001) ))
model.add(layers.Activation(activations.relu))


# 4td Fully Connected Layer
model.add(Dense(units = 10, activation = 'softmax'))



print("[INFO]: Model Summary")

model.summary()

#LEARNING_RATE = 0.0001
LEARNING_RATE = 0.001
MOMENTUM = 0.9

# compile model
opt = SGD(lr=LEARNING_RATE, momentum=MOMENTUM)
model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])

#Log callback
filepath="lr0001.log"
path = F"{filepath}" 
csv_logger = CSVLogger(path, separator=',', append=False)

TRAINING_NUM = train_data.samples #or train_data.n
VALID_NUM = valid_data.samples
EPOCHS = 25

STEP_SIZE_TRAIN = TRAINING_NUM // TRAIN_BATCH_SIZE 
STEP_SIZE_VALID = VALID_NUM // VALID_BATCH_SIZE

#Train Model
history = model.fit_generator(generator = train_data,
                    steps_per_epoch = STEP_SIZE_TRAIN,
                    validation_data = valid_data,
                    validation_steps = STEP_SIZE_VALID,
                    callbacks=[csv_logger],
                    epochs = EPOCHS
)

best_val_acc = max(history.history['val_accuracy'])
print("[INFO] Best Validation Accuracy: %",best_val_acc*100)

#Plot Accuracy and Loss

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)

#accuracy plot
plt.plot(epochs, acc, color='green', label='Training Accuracy')
plt.plot(epochs, val_acc, color='blue', label='Validation Accuracy')
plt.title('Training and Validation Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend()

plt.figure()

#loss plot
plt.plot(epochs, loss, color='pink', label='Training Loss')
plt.plot(epochs, val_loss, color='red', label='Validation Loss')
plt.title('Training and Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.show()

"""# l2 0.0001"""

# Initializing the CNN based AlexNet
model = Sequential()

#valid:zero padding, same:keep same dimensionality by add padding

# Convolution Step 1
model.add(Convolution2D(96, 11, strides = (4, 4), padding = 'valid', input_shape=(227, 227, 3), kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Max Pooling Step 1
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))

# Convolution Step 2
model.add(Convolution2D(256, 5, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Max Pooling Step 2
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding='valid'))

# Convolution Step 3
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Convolution Step 4
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same',  kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Convolution Step 5
model.add(Convolution2D(256, 3, strides=(1,1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))

# Max Pooling Step 3
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))


# Flattening Step --> 6*6*256 = 9216
model.add(Flatten())

# Full Connection Steps
# 1st Fully Connected Layer
model.add(Dense(units = 4096,  kernel_initializer='he_uniform',kernel_regularizer=regularizers.l2(0.0001))) 
model.add(layers.Activation(activations.relu))

# 2nd Fully Connected Layer
model.add(Dense(units = 4096,kernel_initializer='he_uniform', kernel_regularizer=regularizers.l2(0.0001)))
model.add(layers.Activation(activations.relu))


# 3rd Fully Connected Layer
model.add(Dense(units = 1000,  kernel_initializer='he_uniform',kernel_regularizer=regularizers.l2(0.0001) ))
model.add(layers.Activation(activations.relu))


# 4td Fully Connected Layer
model.add(Dense(units = 10, activation = 'softmax'))



print("[INFO]: Model Summary")

model.summary()

#LEARNING_RATE = 0.0001
LEARNING_RATE = 0.001
MOMENTUM = 0.9

# compile model
opt = SGD(lr=LEARNING_RATE, momentum=MOMENTUM)
model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])

#Log callback
filepath="lr00001.log"
path = F"{filepath}" 
csv_logger = CSVLogger(path, separator=',', append=False)

TRAINING_NUM = train_data.samples #or train_data.n
VALID_NUM = valid_data.samples
EPOCHS = 25

STEP_SIZE_TRAIN = TRAINING_NUM // TRAIN_BATCH_SIZE 
STEP_SIZE_VALID = VALID_NUM // VALID_BATCH_SIZE

#Train Model
history = model.fit_generator(generator = train_data,
                    steps_per_epoch = STEP_SIZE_TRAIN,
                    validation_data = valid_data,
                    validation_steps = STEP_SIZE_VALID,
                    callbacks=[csv_logger],
                    epochs = EPOCHS
)

best_val_acc = max(history.history['val_accuracy'])
print("[INFO] Best Validation Accuracy: %",best_val_acc*100)

#Plot Accuracy and Loss

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)

#accuracy plot
plt.plot(epochs, acc, color='green', label='Training Accuracy')
plt.plot(epochs, val_acc, color='blue', label='Validation Accuracy')
plt.title('Training and Validation Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend()

plt.figure()

#loss plot
plt.plot(epochs, loss, color='pink', label='Training Loss')
plt.plot(epochs, val_loss, color='red', label='Validation Loss')
plt.title('Training and Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.show()

"""# l2 (0.001) + data aug"""

# this is the augmentation configuration we will use for training
train_gen = ImageDataGenerator(
rescale = 1./255,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True,
fill_mode='nearest')

#train_gen = ImageDataGenerator(rescale = 1./255)
valid_gen = ImageDataGenerator(rescale = 1./255)

TARGET_SIZE = (227,227)
TRAIN_BATCH_SIZE = 128
VALID_BATCH_SIZE = 32
SEED = 42

#Data Iterator
train_data = train_gen.flow_from_directory(
TRAINING_DIR,
target_size = TARGET_SIZE,
class_mode = 'categorical',
color_mode = "rgb",
batch_size = TRAIN_BATCH_SIZE,
shuffle = True,
seed = SEED
)

valid_data = valid_gen.flow_from_directory(
VALIDATION_DIR,
target_size = TARGET_SIZE,
class_mode = 'categorical',
color_mode = "rgb",
batch_size = VALID_BATCH_SIZE
)

# Initializing the CNN based AlexNet
model = Sequential()
 
#valid:zero padding, same:keep same dimensionality by add padding
 
# Convolution Step 1
model.add(Convolution2D(96, 11, strides = (4, 4), padding = 'valid', input_shape=(227, 227, 3), kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
 
# Max Pooling Step 1
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))
 
# Convolution Step 2
model.add(Convolution2D(256, 5, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
 
# Max Pooling Step 2
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding='valid'))
 
# Convolution Step 3
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
 
# Convolution Step 4
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same',  kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
 
# Convolution Step 5
model.add(Convolution2D(256, 3, strides=(1,1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
 
# Max Pooling Step 3
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))
 
 
# Flattening Step --> 6*6*256 = 9216
model.add(Flatten())
 
# Full Connection Steps
# 1st Fully Connected Layer
model.add(Dense(units = 4096,  kernel_initializer='he_uniform',kernel_regularizer=regularizers.l2(0.001))) 
model.add(layers.Activation(activations.relu))
 
# 2nd Fully Connected Layer
model.add(Dense(units = 4096,kernel_initializer='he_uniform', kernel_regularizer=regularizers.l2(0.001)))
model.add(layers.Activation(activations.relu))
 
 
# 3rd Fully Connected Layer
model.add(Dense(units = 1000,  kernel_initializer='he_uniform',kernel_regularizer=regularizers.l2(0.001) ))
model.add(layers.Activation(activations.relu))
 
 
# 4td Fully Connected Layer
model.add(Dense(units = 10, activation = 'softmax'))
 
 
 
print("[INFO]: Model Summary")
 
model.summary()

from keras.callbacks import CSVLogger
from keras.optimizers import SGD
#LEARNING_RATE = 0.0001
LEARNING_RATE = 0.001
MOMENTUM = 0.9
 
# compile model
opt = SGD(lr=LEARNING_RATE, momentum=MOMENTUM)
model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
 
#Log callback
filepath="lr0-001_data_aug.log"
path = F"{filepath}" 
csv_logger = CSVLogger(path, separator=',', append=False)

TRAINING_NUM = train_data.samples #or train_data.n
VALID_NUM = valid_data.samples
EPOCHS = 25

STEP_SIZE_TRAIN = TRAINING_NUM // TRAIN_BATCH_SIZE 
STEP_SIZE_VALID = VALID_NUM // VALID_BATCH_SIZE

#Train Model
history = model.fit_generator(generator = train_data,
                    steps_per_epoch = STEP_SIZE_TRAIN,
                    validation_data = valid_data,
                    validation_steps = STEP_SIZE_VALID,
                    callbacks=[csv_logger],
                    epochs = EPOCHS
)

best_val_acc = max(history.history['val_accuracy'])
print("[INFO] Best Validation Accuracy: %",best_val_acc*100)

import matplotlib.pyplot as plt
import seaborn as sns
sns.set()

#Plot Accuracy and Loss

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)

#accuracy plot
plt.plot(epochs, acc, color='green', label='Training Accuracy')
plt.plot(epochs, val_acc, color='blue', label='Validation Accuracy')
plt.title('Training and Validation Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend()

plt.figure()

#loss plot
plt.plot(epochs, loss, color='pink', label='Training Loss')
plt.plot(epochs, val_loss, color='red', label='Validation Loss')
plt.title('Training and Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.show()

"""# LN before RELU"""

# this is the augmentation configuration we will use for training
train_gen = ImageDataGenerator(rescale = 1./255)
valid_gen = ImageDataGenerator(rescale = 1./255)

TARGET_SIZE = (227,227)
TRAIN_BATCH_SIZE = 128
VALID_BATCH_SIZE = 32
SEED = 42
 
#Data Iterator
train_data = train_gen.flow_from_directory(
TRAINING_DIR,
target_size = TARGET_SIZE,
class_mode = 'categorical',
color_mode = "rgb",
batch_size = TRAIN_BATCH_SIZE,
shuffle = True,
seed = SEED
)
 
valid_data = valid_gen.flow_from_directory(
VALIDATION_DIR,
target_size = TARGET_SIZE,
class_mode = 'categorical',
color_mode = "rgb",
batch_size = VALID_BATCH_SIZE
)

# Initializing the CNN based AlexNet
model = Sequential()

#valid:zero padding, same:keep same dimensionality by add padding

# Convolution Step 1
model.add(Convolution2D(96, 11, strides = (4, 4), padding = 'valid', input_shape=(227, 227, 3), kernel_initializer='he_uniform'))
tf.keras.layers.LayerNormalization()
model.add(layers.Activation(activations.relu))

# Max Pooling Step 1
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))

# Convolution Step 2
model.add(Convolution2D(256, 5, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
tf.keras.layers.LayerNormalization()
model.add(layers.Activation(activations.relu))

# Max Pooling Step 2
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding='valid'))

# Convolution Step 3
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
tf.keras.layers.LayerNormalization()
model.add(layers.Activation(activations.relu))

# Convolution Step 4
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same',  kernel_initializer='he_uniform'))
tf.keras.layers.LayerNormalization()
model.add(layers.Activation(activations.relu))

# Convolution Step 5
model.add(Convolution2D(256, 3, strides=(1,1), padding='same', kernel_initializer='he_uniform'))
tf.keras.layers.LayerNormalization()
model.add(layers.Activation(activations.relu))

# Max Pooling Step 3
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))


# Flattening Step --> 6*6*256 = 9216
model.add(Flatten())

# Full Connection Steps
# 1st Fully Connected Layer
model.add(Dense(units = 4096,  kernel_initializer='he_uniform',kernel_regularizer=regularizers.l2(0.0001))) 
tf.keras.layers.LayerNormalization()
model.add(layers.Activation(activations.relu))


# 2nd Fully Connected Layer
model.add(Dense(units = 4096,kernel_initializer='he_uniform', kernel_regularizer=regularizers.l2(0.0001)))
tf.keras.layers.LayerNormalization()
model.add(layers.Activation(activations.relu))



# 3rd Fully Connected Layer
model.add(Dense(units = 1000,  kernel_initializer='he_uniform',kernel_regularizer=regularizers.l2(0.0001) ))
tf.keras.layers.LayerNormalization()
model.add(layers.Activation(activations.relu))


# 4td Fully Connected Layer
model.add(Dense(units = 10, activation = 'softmax'))



print("[INFO]: Model Summary")

model.summary()

#LEARNING_RATE = 0.0001
LEARNING_RATE = 0.001
MOMENTUM = 0.9

# compile model
opt = SGD(lr=LEARNING_RATE, momentum=MOMENTUM)
model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])

#Log callback
filepath="lnbefore_relu.log"
path = F"{filepath}" 
csv_logger = CSVLogger(path, separator=',', append=False)

TRAINING_NUM = train_data.samples #or train_data.n
VALID_NUM = valid_data.samples
EPOCHS = 25

STEP_SIZE_TRAIN = TRAINING_NUM // TRAIN_BATCH_SIZE 
STEP_SIZE_VALID = VALID_NUM // VALID_BATCH_SIZE

#Train Model
history = model.fit_generator(generator = train_data,
                    steps_per_epoch = STEP_SIZE_TRAIN,
                    validation_data = valid_data,
                    validation_steps = STEP_SIZE_VALID,
                    callbacks=[csv_logger],
                    epochs = EPOCHS
)

best_val_acc = max(history.history['val_accuracy'])
print("[INFO] Best Validation Accuracy: %",best_val_acc*100)

#Plot Accuracy and Loss

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)

#accuracy plot
plt.plot(epochs, acc, color='green', label='Training Accuracy')
plt.plot(epochs, val_acc, color='blue', label='Validation Accuracy')
plt.title('Training and Validation Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend()

plt.figure()

#loss plot
plt.plot(epochs, loss, color='pink', label='Training Loss')
plt.plot(epochs, val_loss, color='red', label='Validation Loss')
plt.title('Training and Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.show()

"""# LN after RELU"""

# Initializing the CNN based AlexNet
model = Sequential()
 
#valid:zero padding, same:keep same dimensionality by add padding
 
# Convolution Step 1
model.add(Convolution2D(96, 11, strides = (4, 4), padding = 'valid', input_shape=(227, 227, 3), kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.LayerNormalization()
 
# Max Pooling Step 1
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))
 
# Convolution Step 2
model.add(Convolution2D(256, 5, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.LayerNormalization()
 
# Max Pooling Step 2
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding='valid'))
 
# Convolution Step 3
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.LayerNormalization()
 
# Convolution Step 4
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same',  kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.LayerNormalization()
 
# Convolution Step 5
model.add(Convolution2D(256, 3, strides=(1,1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.LayerNormalization()
 
# Max Pooling Step 3
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))
 
 
# Flattening Step --> 6*6*256 = 9216
model.add(Flatten())
 
# Full Connection Steps
# 1st Fully Connected Layer
model.add(Dense(units = 4096,  kernel_initializer='he_uniform',kernel_regularizer=regularizers.l2(0.0001))) 
model.add(layers.Activation(activations.relu))
tf.keras.layers.LayerNormalization()
 
# 2nd Fully Connected Layer
model.add(Dense(units = 4096,kernel_initializer='he_uniform', kernel_regularizer=regularizers.l2(0.0001)))
model.add(layers.Activation(activations.relu))
tf.keras.layers.LayerNormalization()
 
 
# 3rd Fully Connected Layer
model.add(Dense(units = 1000,  kernel_initializer='he_uniform',kernel_regularizer=regularizers.l2(0.0001) ))
model.add(layers.Activation(activations.relu))
tf.keras.layers.LayerNormalization()
 
# 4td Fully Connected Layer
model.add(Dense(units = 10, activation = 'softmax'))
 
 
 
print("[INFO]: Model Summary")
 
model.summary()

#LEARNING_RATE = 0.0001
LEARNING_RATE = 0.001
MOMENTUM = 0.9
 
# compile model
opt = SGD(lr=LEARNING_RATE, momentum=MOMENTUM)
model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
 
#Log callback
filepath="lnafter_relu.log"
path = F"{filepath}" 
csv_logger = CSVLogger(path, separator=',', append=False)

TRAINING_NUM = train_data.samples #or train_data.n
VALID_NUM = valid_data.samples
EPOCHS = 25
 
STEP_SIZE_TRAIN = TRAINING_NUM // TRAIN_BATCH_SIZE 
STEP_SIZE_VALID = VALID_NUM // VALID_BATCH_SIZE
 
#Train Model
history = model.fit_generator(generator = train_data,
                    steps_per_epoch = STEP_SIZE_TRAIN,
                    validation_data = valid_data,
                    validation_steps = STEP_SIZE_VALID,
                    callbacks=[csv_logger],
                    epochs = EPOCHS
)

best_val_acc = max(history.history['val_accuracy'])
print("[INFO] Best Validation Accuracy: %",best_val_acc*100)

#Plot Accuracy and Loss
 
import matplotlib.pyplot as plt
 
acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)
 
#accuracy plot
plt.plot(epochs, acc, color='green', label='Training Accuracy')
plt.plot(epochs, val_acc, color='blue', label='Validation Accuracy')
plt.title('Training and Validation Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend()
 
plt.figure()
 
#loss plot
plt.plot(epochs, loss, color='pink', label='Training Loss')
plt.plot(epochs, val_loss, color='red', label='Validation Loss')
plt.title('Training and Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
 
plt.show()

"""# BN before RELU"""

# Initializing the CNN based AlexNet
model = Sequential()
 
#valid:zero padding, same:keep same dimensionality by add padding
 
# Convolution Step 1
model.add(Convolution2D(96, 11, strides = (4, 4), padding = 'valid', input_shape=(227, 227, 3), kernel_initializer='he_uniform'))
tf.keras.layers.BatchNormalization()
model.add(layers.Activation(activations.relu))
 
# Max Pooling Step 1
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))
 
# Convolution Step 2
model.add(Convolution2D(256, 5, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
tf.keras.layers.BatchNormalization()
model.add(layers.Activation(activations.relu))
 
# Max Pooling Step 2
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding='valid'))
 
# Convolution Step 3
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
tf.keras.layers.BatchNormalization()
model.add(layers.Activation(activations.relu))
 
# Convolution Step 4
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same',  kernel_initializer='he_uniform'))
tf.keras.layers.BatchNormalization()
model.add(layers.Activation(activations.relu))
 
# Convolution Step 5
model.add(Convolution2D(256, 3, strides=(1,1), padding='same', kernel_initializer='he_uniform'))
tf.keras.layers.BatchNormalization()
model.add(layers.Activation(activations.relu))
 
# Max Pooling Step 3
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))
 
 
# Flattening Step --> 6*6*256 = 9216
model.add(Flatten())
 
# Full Connection Steps
# 1st Fully Connected Layer
model.add(Dense(units = 4096,  kernel_initializer='he_uniform',kernel_regularizer=regularizers.l2(0.0001))) 
tf.keras.layers.BatchNormalization()
model.add(layers.Activation(activations.relu))
 
 
# 2nd Fully Connected Layer
model.add(Dense(units = 4096,kernel_initializer='he_uniform', kernel_regularizer=regularizers.l2(0.0001)))
tf.keras.layers.BatchNormalization()
model.add(layers.Activation(activations.relu))
 
 
 
# 3rd Fully Connected Layer
model.add(Dense(units = 1000,  kernel_initializer='he_uniform',kernel_regularizer=regularizers.l2(0.0001) ))
tf.keras.layers.BatchNormalization()
model.add(layers.Activation(activations.relu))
 
 
# 4td Fully Connected Layer
model.add(Dense(units = 10, activation = 'softmax'))
 
 
 
print("[INFO]: Model Summary")
 
model.summary()

#LEARNING_RATE = 0.0001
LEARNING_RATE = 0.001
MOMENTUM = 0.9
 
# compile model
opt = SGD(lr=LEARNING_RATE, momentum=MOMENTUM)
model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
 
#Log callback
filepath="bnbefore_relu.log"
path = F"{filepath}" 
csv_logger = CSVLogger(path, separator=',', append=False)

TRAINING_NUM = train_data.samples #or train_data.n
VALID_NUM = valid_data.samples
EPOCHS = 25
 
STEP_SIZE_TRAIN = TRAINING_NUM // TRAIN_BATCH_SIZE 
STEP_SIZE_VALID = VALID_NUM // VALID_BATCH_SIZE
 
#Train Model
history = model.fit_generator(generator = train_data,
                    steps_per_epoch = STEP_SIZE_TRAIN,
                    validation_data = valid_data,
                    validation_steps = STEP_SIZE_VALID,
                    callbacks=[csv_logger],
                    epochs = EPOCHS
)

best_val_acc = max(history.history['val_accuracy'])
print("[INFO] Best Validation Accuracy: %",best_val_acc*100)

#Plot Accuracy and Loss

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)

#accuracy plot
plt.plot(epochs, acc, color='green', label='Training Accuracy')
plt.plot(epochs, val_acc, color='blue', label='Validation Accuracy')
plt.title('Training and Validation Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend()

plt.figure()

#loss plot
plt.plot(epochs, loss, color='pink', label='Training Loss')
plt.plot(epochs, val_loss, color='red', label='Validation Loss')
plt.title('Training and Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.show()

"""# BN after RELU"""

# Initializing the CNN based AlexNet
model = Sequential()

#valid:zero padding, same:keep same dimensionality by add padding

# Convolution Step 1
model.add(Convolution2D(96, 11, strides = (4, 4), padding = 'valid', input_shape=(227, 227, 3), kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Max Pooling Step 1
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))

# Convolution Step 2
model.add(Convolution2D(256, 5, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Max Pooling Step 2
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding='valid'))

# Convolution Step 3
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Convolution Step 4
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same',  kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Convolution Step 5
model.add(Convolution2D(256, 3, strides=(1,1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Max Pooling Step 3
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))


# Flattening Step --> 6*6*256 = 9216
model.add(Flatten())

# Full Connection Steps
# 1st Fully Connected Layer
model.add(Dense(units = 4096,  kernel_initializer='he_uniform',kernel_regularizer=regularizers.l2(0.0001))) 
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# 2nd Fully Connected Layer
model.add(Dense(units = 4096,kernel_initializer='he_uniform', kernel_regularizer=regularizers.l2(0.0001)))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()


# 3rd Fully Connected Layer
model.add(Dense(units = 1000,  kernel_initializer='he_uniform',kernel_regularizer=regularizers.l2(0.0001) ))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# 4td Fully Connected Layer
model.add(Dense(units = 10, activation = 'softmax'))



print("[INFO]: Model Summary")

model.summary()

#LEARNING_RATE = 0.0001
LEARNING_RATE = 0.001
MOMENTUM = 0.9

# compile model
opt = SGD(lr=LEARNING_RATE, momentum=MOMENTUM)
model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])

#Log callback
filepath="bnafter_relulog"
path = F"{filepath}" 
csv_logger = CSVLogger(path, separator=',', append=False)

TRAINING_NUM = train_data.samples #or train_data.n
VALID_NUM = valid_data.samples
EPOCHS = 25

STEP_SIZE_TRAIN = TRAINING_NUM // TRAIN_BATCH_SIZE 
STEP_SIZE_VALID = VALID_NUM // VALID_BATCH_SIZE

#Train Model
history = model.fit_generator(generator = train_data,
                    steps_per_epoch = STEP_SIZE_TRAIN,
                    validation_data = valid_data,
                    validation_steps = STEP_SIZE_VALID,
                    callbacks=[csv_logger],
                    epochs = EPOCHS
)

best_val_acc = max(history.history['val_accuracy'])
print("[INFO] Best Validation Accuracy: %",best_val_acc*100)

#Plot Accuracy and Loss

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)

#accuracy plot
plt.plot(epochs, acc, color='green', label='Training Accuracy')
plt.plot(epochs, val_acc, color='blue', label='Validation Accuracy')
plt.title('Training and Validation Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend()

plt.figure()

#loss plot
plt.plot(epochs, loss, color='pink', label='Training Loss')
plt.plot(epochs, val_loss, color='red', label='Validation Loss')
plt.title('Training and Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.show()

"""# 50 Epoch"""

# Initializing the CNN based AlexNet
model = Sequential()

#valid:zero padding, same:keep same dimensionality by add padding

# Convolution Step 1
model.add(Convolution2D(96, 11, strides = (4, 4), padding = 'valid', input_shape=(227, 227, 3), kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Max Pooling Step 1
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))

# Convolution Step 2
model.add(Convolution2D(256, 5, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Max Pooling Step 2
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding='valid'))

# Convolution Step 3
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Convolution Step 4
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same',  kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Convolution Step 5
model.add(Convolution2D(256, 3, strides=(1,1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Max Pooling Step 3
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))


# Flattening Step --> 6*6*256 = 9216
model.add(Flatten())

# Full Connection Steps
# 1st Fully Connected Layer
model.add(Dense(units = 4096,  kernel_initializer='he_uniform',kernel_regularizer=regularizers.l2(0.0001))) 
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# 2nd Fully Connected Layer
model.add(Dense(units = 4096,kernel_initializer='he_uniform', kernel_regularizer=regularizers.l2(0.0001)))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()


# 3rd Fully Connected Layer
model.add(Dense(units = 1000,  kernel_initializer='he_uniform',kernel_regularizer=regularizers.l2(0.0001) ))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# 4td Fully Connected Layer
model.add(Dense(units = 10, activation = 'softmax'))



print("[INFO]: Model Summary")

model.summary()

#LEARNING_RATE = 0.0001
LEARNING_RATE = 0.001
MOMENTUM = 0.9

# compile model
opt = SGD(lr=LEARNING_RATE, momentum=MOMENTUM)
model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])

#Log callback
filepath="50epoch.log"
path = F"{filepath}" 
csv_logger = CSVLogger(path, separator=',', append=False)

TRAINING_NUM = train_data.samples #or train_data.n
VALID_NUM = valid_data.samples
EPOCHS = 50

STEP_SIZE_TRAIN = TRAINING_NUM // TRAIN_BATCH_SIZE 
STEP_SIZE_VALID = VALID_NUM // VALID_BATCH_SIZE

#Train Model
history = model.fit_generator(generator = train_data,
                    steps_per_epoch = STEP_SIZE_TRAIN,
                    validation_data = valid_data,
                    validation_steps = STEP_SIZE_VALID,
                    callbacks=[csv_logger],
                    epochs = EPOCHS
)

best_val_acc = max(history.history['val_accuracy'])
print("[INFO] Best Validation Accuracy: %",best_val_acc*100)

#Plot Accuracy and Loss

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)

#accuracy plot
plt.plot(epochs, acc, color='green', label='Training Accuracy')
plt.plot(epochs, val_acc, color='blue', label='Validation Accuracy')
plt.title('Training and Validation Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend()

plt.figure()

#loss plot
plt.plot(epochs, loss, color='pink', label='Training Loss')
plt.plot(epochs, val_loss, color='red', label='Validation Loss')
plt.title('Training and Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.show()

"""# 50 Epoch + Data Aug"""

# this is the augmentation configuration we will use for training
train_gen = ImageDataGenerator(
rescale = 1./255,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True,
fill_mode='nearest')

#train_gen = ImageDataGenerator(rescale = 1./255)
valid_gen = ImageDataGenerator(rescale = 1./255)

TARGET_SIZE = (227,227)
TRAIN_BATCH_SIZE = 128
VALID_BATCH_SIZE = 32
SEED = 42

#Data Iterator
train_data = train_gen.flow_from_directory(
TRAINING_DIR,
target_size = TARGET_SIZE,
class_mode = 'categorical',
color_mode = "rgb",
batch_size = TRAIN_BATCH_SIZE,
shuffle = True,
seed = SEED
)

valid_data = valid_gen.flow_from_directory(
VALIDATION_DIR,
target_size = TARGET_SIZE,
class_mode = 'categorical',
color_mode = "rgb",
batch_size = VALID_BATCH_SIZE
)

# Initializing the CNN based AlexNet
model = Sequential()

#valid:zero padding, same:keep same dimensionality by add padding

# Convolution Step 1
model.add(Convolution2D(96, 11, strides = (4, 4), padding = 'valid', input_shape=(227, 227, 3), kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Max Pooling Step 1
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))

# Convolution Step 2
model.add(Convolution2D(256, 5, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Max Pooling Step 2
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding='valid'))

# Convolution Step 3
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Convolution Step 4
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same',  kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Convolution Step 5
model.add(Convolution2D(256, 3, strides=(1,1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Max Pooling Step 3
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))


# Flattening Step --> 6*6*256 = 9216
model.add(Flatten())

# Full Connection Steps
# 1st Fully Connected Layer
model.add(Dense(units = 4096,  kernel_initializer='he_uniform',kernel_regularizer=regularizers.l2(0.0001))) 
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# 2nd Fully Connected Layer
model.add(Dense(units = 4096,kernel_initializer='he_uniform', kernel_regularizer=regularizers.l2(0.0001)))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()


# 3rd Fully Connected Layer
model.add(Dense(units = 1000,  kernel_initializer='he_uniform',kernel_regularizer=regularizers.l2(0.0001) ))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# 4td Fully Connected Layer
model.add(Dense(units = 10, activation = 'softmax'))



print("[INFO]: Model Summary")

model.summary()

#LEARNING_RATE = 0.0001
LEARNING_RATE = 0.001
MOMENTUM = 0.9

# compile model
opt = SGD(lr=LEARNING_RATE, momentum=MOMENTUM)
model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])

#Log callback
filepath="50epoch_dataaug.log"
path = F"{filepath}" 
csv_logger = CSVLogger(path, separator=',', append=False)

TRAINING_NUM = train_data.samples #or train_data.n
VALID_NUM = valid_data.samples
EPOCHS = 50

STEP_SIZE_TRAIN = TRAINING_NUM // TRAIN_BATCH_SIZE 
STEP_SIZE_VALID = VALID_NUM // VALID_BATCH_SIZE

#Train Model
history = model.fit_generator(generator = train_data,
                    steps_per_epoch = STEP_SIZE_TRAIN,
                    validation_data = valid_data,
                    validation_steps = STEP_SIZE_VALID,
                    callbacks=[csv_logger],
                    epochs = EPOCHS
)

best_val_acc = max(history.history['val_accuracy'])
print("[INFO] Best Validation Accuracy: %",best_val_acc*100)

#Plot Accuracy and Loss

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)

#accuracy plot
plt.plot(epochs, acc, color='green', label='Training Accuracy')
plt.plot(epochs, val_acc, color='blue', label='Validation Accuracy')
plt.title('Training and Validation Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend()

plt.figure()

#loss plot
plt.plot(epochs, loss, color='pink', label='Training Loss')
plt.plot(epochs, val_loss, color='red', label='Validation Loss')
plt.title('Training and Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.show()



"""# 100 Epoch + Data Aug"""

# Initializing the CNN based AlexNet
model = Sequential()

#valid:zero padding, same:keep same dimensionality by add padding

# Convolution Step 1
model.add(Convolution2D(96, 11, strides = (4, 4), padding = 'valid', input_shape=(227, 227, 3), kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Max Pooling Step 1
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))

# Convolution Step 2
model.add(Convolution2D(256, 5, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Max Pooling Step 2
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding='valid'))

# Convolution Step 3
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Convolution Step 4
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same',  kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Convolution Step 5
model.add(Convolution2D(256, 3, strides=(1,1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Max Pooling Step 3
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))


# Flattening Step --> 6*6*256 = 9216
model.add(Flatten())

# Full Connection Steps
# 1st Fully Connected Layer
model.add(Dense(units = 4096,  kernel_initializer='he_uniform',kernel_regularizer=regularizers.l2(0.0001))) 
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# 2nd Fully Connected Layer
model.add(Dense(units = 4096,kernel_initializer='he_uniform', kernel_regularizer=regularizers.l2(0.0001)))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()


# 3rd Fully Connected Layer
model.add(Dense(units = 1000,  kernel_initializer='he_uniform',kernel_regularizer=regularizers.l2(0.0001) ))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# 4td Fully Connected Layer
model.add(Dense(units = 10, activation = 'softmax'))



print("[INFO]: Model Summary")

model.summary()

#LEARNING_RATE = 0.0001
LEARNING_RATE = 0.001
MOMENTUM = 0.9

# compile model
opt = SGD(lr=LEARNING_RATE, momentum=MOMENTUM)
model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])

#Log callback
filepath="50epoch.log"
path = F"{filepath}" 
csv_logger = CSVLogger(path, separator=',', append=False)

TRAINING_NUM = train_data.samples #or train_data.n
VALID_NUM = valid_data.samples
EPOCHS = 100

STEP_SIZE_TRAIN = TRAINING_NUM // TRAIN_BATCH_SIZE 
STEP_SIZE_VALID = VALID_NUM // VALID_BATCH_SIZE

#Train Model
history = model.fit_generator(generator = train_data,
                    steps_per_epoch = STEP_SIZE_TRAIN,
                    validation_data = valid_data,
                    validation_steps = STEP_SIZE_VALID,
                    callbacks=[csv_logger],
                    epochs = EPOCHS
)

best_val_acc = max(history.history['val_accuracy'])
print("[INFO] Best Validation Accuracy: %",best_val_acc*100)

#Plot Accuracy and Loss

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)

#accuracy plot
plt.plot(epochs, acc, color='green', label='Training Accuracy')
plt.plot(epochs, val_acc, color='blue', label='Validation Accuracy')
plt.title('Training and Validation Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend()

plt.figure()

#loss plot
plt.plot(epochs, loss, color='pink', label='Training Loss')
plt.plot(epochs, val_loss, color='red', label='Validation Loss')
plt.title('Training and Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.show()

"""# L2 yerine Dropout +  100 Epoch"""

# Initializing the CNN based AlexNet
model = Sequential()

#valid:zero padding, same:keep same dimensionality by add padding

# Convolution Step 1
model.add(Convolution2D(96, 11, strides = (4, 4), padding = 'valid', input_shape=(227, 227, 3), kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Max Pooling Step 1
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))

# Convolution Step 2
model.add(Convolution2D(256, 5, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Max Pooling Step 2
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding='valid'))

# Convolution Step 3
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Convolution Step 4
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same',  kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Convolution Step 5
model.add(Convolution2D(256, 3, strides=(1,1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Max Pooling Step 3
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))


# Flattening Step --> 6*6*256 = 9216
model.add(Flatten())

# Full Connection Steps
# 1st Fully Connected Layer
model.add(Dense(units = 4096,  kernel_initializer='he_uniform')) 
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()
model.add(Dropout(0.2))

# 2nd Fully Connected Layer
model.add(Dense(units = 4096,kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()
model.add(Dropout(0.2))


# 3rd Fully Connected Layer
model.add(Dense(units = 1000,  kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()
model.add(Dropout(0.2))

# 4td Fully Connected Layer
model.add(Dense(units = 10, activation = 'softmax'))



print("[INFO]: Model Summary")

model.summary()

#LEARNING_RATE = 0.0001
LEARNING_RATE = 0.001
MOMENTUM = 0.9

# compile model
opt = SGD(lr=LEARNING_RATE, momentum=MOMENTUM)
model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])

#Log callback
filepath="50epoch.log"
path = F"{filepath}" 
csv_logger = CSVLogger(path, separator=',', append=False)

TRAINING_NUM = train_data.samples #or train_data.n
VALID_NUM = valid_data.samples
EPOCHS = 100

STEP_SIZE_TRAIN = TRAINING_NUM // TRAIN_BATCH_SIZE 
STEP_SIZE_VALID = VALID_NUM // VALID_BATCH_SIZE

#Train Model
history = model.fit_generator(generator = train_data,
                    steps_per_epoch = STEP_SIZE_TRAIN,
                    validation_data = valid_data,
                    validation_steps = STEP_SIZE_VALID,
                    callbacks=[csv_logger],
                    epochs = EPOCHS
)

best_val_acc = max(history.history['val_accuracy'])
print("[INFO] Best Validation Accuracy: %",best_val_acc*100)

#Plot Accuracy and Loss

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)

#accuracy plot
plt.plot(epochs, acc, color='green', label='Training Accuracy')
plt.plot(epochs, val_acc, color='blue', label='Validation Accuracy')
plt.title('Training and Validation Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend()

plt.figure()

#loss plot
plt.plot(epochs, loss, color='pink', label='Training Loss')
plt.plot(epochs, val_loss, color='red', label='Validation Loss')
plt.title('Training and Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.show()



"""# Without 1000 dense layer"""

# Initializing the CNN based AlexNet
model = Sequential()

#valid:zero padding, same:keep same dimensionality by add padding

# Convolution Step 1
model.add(Convolution2D(96, 11, strides = (4, 4), padding = 'valid', input_shape=(227, 227, 3), kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Max Pooling Step 1
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))

# Convolution Step 2
model.add(Convolution2D(256, 5, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Max Pooling Step 2
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding='valid'))

# Convolution Step 3
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Convolution Step 4
model.add(Convolution2D(384, 3, strides = (1, 1), padding='same',  kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Convolution Step 5
model.add(Convolution2D(256, 3, strides=(1,1), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()

# Max Pooling Step 3
model.add(MaxPooling2D(pool_size = (3, 3), strides = (2, 2), padding = 'valid'))


# Flattening Step --> 6*6*256 = 9216
model.add(Flatten())

# Full Connection Steps
# 1st Fully Connected Layer
model.add(Dense(units = 4096,  kernel_initializer='he_uniform')) 
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()
model.add(Dropout(0.2))

# 2nd Fully Connected Layer
model.add(Dense(units = 4096,kernel_initializer='he_uniform'))
model.add(layers.Activation(activations.relu))
tf.keras.layers.BatchNormalization()
model.add(Dropout(0.2))




# 3th Fully Connected Layer
model.add(Dense(units = 10, activation = 'softmax'))



print("[INFO]: Model Summary")

model.summary()

#LEARNING_RATE = 0.0001
LEARNING_RATE = 0.001
MOMENTUM = 0.9

# compile model
opt = SGD(lr=LEARNING_RATE, momentum=MOMENTUM)
model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])

#Log callback
filepath="50epoch.log"
path = F"{filepath}" 
csv_logger = CSVLogger(path, separator=',', append=False)

TRAINING_NUM = train_data.samples #or train_data.n
VALID_NUM = valid_data.samples
EPOCHS = 100

STEP_SIZE_TRAIN = TRAINING_NUM // TRAIN_BATCH_SIZE 
STEP_SIZE_VALID = VALID_NUM // VALID_BATCH_SIZE

#Train Model
history = model.fit_generator(generator = train_data,
                    steps_per_epoch = STEP_SIZE_TRAIN,
                    validation_data = valid_data,
                    validation_steps = STEP_SIZE_VALID,
                    callbacks=[csv_logger],
                    epochs = EPOCHS
)

best_val_acc = max(history.history['val_accuracy'])
print("[INFO] Best Validation Accuracy: %",best_val_acc*100)



"""# 25 epoch +  dropout + data aug"""

TRAINING_NUM = train_data.samples #or train_data.n
VALID_NUM = valid_data.samples
EPOCHS = 25

STEP_SIZE_TRAIN = TRAINING_NUM // TRAIN_BATCH_SIZE 
STEP_SIZE_VALID = VALID_NUM // VALID_BATCH_SIZE

#Train Model
history = model.fit_generator(generator = train_data,
                    steps_per_epoch = STEP_SIZE_TRAIN,
                    validation_data = valid_data,
                    validation_steps = STEP_SIZE_VALID,
                    callbacks=[csv_logger],
                    epochs = EPOCHS
)

best_val_acc = max(history.history['val_accuracy'])
print("[INFO] Best Validation Accuracy: %",best_val_acc*100)