maja1032_ml_final_2.py

# -*- coding: utf-8 -*-
"""maja1032_ML-Final_2.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1aM6JLaxuOlXL77b2C7OzEnwA-ZcxsBpB
"""

get_ipython().system('git clone https://github.com/JaMa-95/ML-Final.git')

"""# Final Projext Machine Learning in Python
**Jakob Mattes**

**Maja1032@hs-karlsruhe.de**

**75269**

Made with google colab

"""

import numpy as np
import pandas as pd
import os
import cv2
import random
from tqdm import tqdm

from tensorflow.keras import applications
from tensorflow.keras.models import Sequential, Model
from tensorflow.keras.layers import Dense, Conv2D, MaxPool2D, Flatten, BatchNormalization, Dropout, MaxPool3D
from tensorflow.keras.preprocessing.image import ImageDataGenerator

#library np_utils to make labels categorical
from tensorflow.python.keras.utils import np_utils

from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split

from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras import optimizers

import matplotlib.pyplot as plt

categories = ['dog', 'panda', 'cat']
X_train, X_test = [], []
y_train, y_test = [], []
imagePaths = []
HEIGHT = 32
WIDTH = 55
N_CHANNELS = 3

########################################################################
########Changed the file path because of google colab###################
########################################################################

# load training data
for k, category in enumerate(categories):
    for f in os.listdir('/content/ML-Final/data/train/' + category):
        imagePaths.append(['/content/ML-Final/data/train/' + category+'/'+f, k])

# loop over the input images
for imagePath in tqdm(imagePaths):
    if 'ds_store' in imagePath[0].lower():
        continue
    # load the image, resize the image to be HEIGHT * WIDTH pixels (ignoring
    # aspect ratio) and store the image in the data list
    image = cv2.imread(imagePath[0])
    image = cv2.resize(image, (WIDTH, HEIGHT))  # .flatten()
    X_train.append(image)
    # extract the class label from the image path and update the
    # labels list
    label = imagePath[1]
    y_train.append(label)




# load test data
imagePaths = []
for k, category in enumerate(categories):
    for f in os.listdir('/content/ML-Final/data/test/' + category):
        imagePaths.append(['/content/ML-Final/data/test/' + category+'/'+f, k])

# loop over the input images
for imagePath in tqdm(imagePaths):
    if 'ds_store' in imagePath[0].lower():
        continue
    # load the image, resize the image to be HEIGHT * WIDTH pixels (ignoring
    # aspect ratio) and store the image in the data list
    image = cv2.imread(imagePath[0])
    image = cv2.resize(image, (WIDTH, HEIGHT))  # .flatten()
    X_test.append(image)
    # extract the class label from the image path and update the
    # labels list
    label = imagePath[1]
    y_test.append(label)


X_train, X_test, y_train, y_test  = np.array(X_train), np.array(X_test), np.array(y_train), np.array(y_test)


############
# Create here your code. X_train, X_test, and so on is already given and splitted.

#Make y_train and y_test categorical (three dimensional vector)
y_train = np_utils.to_categorical(y_train)
y_test = np_utils.to_categorical(y_test)

#DataGenerator
datagen = ImageDataGenerator(
    featurewise_center=False, 
    samplewise_center=False,
    featurewise_std_normalization=False, 
    samplewise_std_normalization=False,
    zca_whitening=False, 
    zca_epsilon=1e-06, 
    rotation_range=30, 
    width_shift_range=0.1,
    height_shift_range=0.1, 
    brightness_range=None, 
    shear_range=0.0, 
    zoom_range=0.2,
    channel_shift_range=0.0, 
    fill_mode='nearest', 
    cval=0.0,
    horizontal_flip=True, 
    vertical_flip=True, 
    rescale=1./255,
    preprocessing_function=None, 
    data_format=None, 
    validation_split=0.0, 
    dtype=None
)


datagen_test = ImageDataGenerator(rescale=1./255, samplewise_center=True)

#Fit datagen to training data
datagen.fit(X_train)

# Tried to normalize data but no better progress
#X_train = np.array(X_train)/255
#X_test = np.array(X_test)/255

#y_train = np.array(y_train)/255
#y_test = np.array(y_test)/255
X_train.shape

#train_flow = datagen.flow_from_dataframe(X_train, x_col ='Filepath', y_col ='Target', target_size=(WIDTH, HEIGHT),interpolation='lanczos', validate_filenames = False)

"""# **Self Made Model Start here**

**Selfmade Model 1**

Good Accuracy/ Bad Val_Accuracy
"""

# Define our CNN
model = Sequential()
model.add(Conv2D(32, (3,3), strides=1, padding='same', activation='relu', input_shape=X_train.shape[1:],kernel_regularizer='l2')) # this layer has 32 filters. The filter size is: (3,3)
model.add(BatchNormalization()) # This is optional to avoid overfitting
model.add(MaxPool2D((2,2), strides=2, padding='same'))

model.add(Conv2D(64 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.1)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))

model.add(Conv2D(64 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(BatchNormalization())
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))

model.add(Conv2D(128 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.2))
model.add(BatchNormalization())
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))

model.add(Conv2D(256 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.2))
model.add(BatchNormalization())
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))


model.add(Flatten())
model.add(Dense(128, activation='relu'))

# Use softmax and 3 ouput layers because of three labels
model.add(Dense(3, activation='softmax')) 

#apply learning rate to adam optimizer
opt = optimizers.Adam(learning_rate=0.00001)
#apply decay rate to optimizer
lr_schedule = optimizers.schedules.ExponentialDecay(
    initial_learning_rate=1e-5,
    decay_steps=10000,
    decay_rate=0.9)
optimizer = optimizers.Adam(learning_rate=lr_schedule)

model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy']) # make sure we have categorical_crossentropy as loss
model.summary()

#Early stopping (not used)
es_callback = EarlyStopping(monitor='val_loss', patience=3)

#fit call to use the datagen. with 1000 epochs
diagramm = model.fit(datagen.flow(X_train,y_train, batch_size = 35) ,epochs = 1000 , validation_data = (X_test, y_test))

#Evaluation part; printing accuracy
y_pred_model1 = np.argmax(model.predict(X_test), axis=-1)
y_test_model1 = np.argmax(y_test, axis=-1)
y_train_model1 = np.argmax(y_train, axis=-1)
print('Train Accuracy of the model is', accuracy_score(y_train_model1, np.argmax(model.predict(X_train), axis=-1)))
print('Test Accuracy of the model is', accuracy_score(y_test_model1, y_pred_model1))



# Plot accuracy graph
plt.plot(diagramm.history['accuracy'])
plt.plot(diagramm.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()

# Plot loss graph
plt.plot(diagramm.history['loss'])
plt.plot(diagramm.history['val_loss'])
plt.title('model accuracy')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()

"""**Selfmade Model 2**

Fewer Neurons
"""

# Define our CNN
model = Sequential()
model.add(Conv2D(32, (3,3), strides=1, padding='same', activation='relu', input_shape=X_train.shape[1:],kernel_regularizer='l2')) # this layer has 32 filters. The filter size is: (3,3)
model.add(BatchNormalization()) # This is optional to avoid overfitting
model.add(MaxPool2D((2,2), strides=2, padding='same'))

model.add(Conv2D(40 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.1)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))

model.add(Conv2D(128 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.6)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))

model.add(Conv2D(54 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.4)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))

model.add(Conv2D(40 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.1)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))


model.add(Flatten())
model.add(Dense(32, activation='relu'))

# Use softmax and 3 ouput layers because of three labels
model.add(Dense(3, activation='softmax')) 

#apply learning rate to adam optimizer
opt = optimizers.Adam(learning_rate=0.0001)
#apply decay rate to optimizer
lr_schedule = optimizers.schedules.ExponentialDecay(
    initial_learning_rate=1e-5,
    decay_steps=10000,
    decay_rate=0.9)
optimizer = optimizers.Adam(learning_rate=lr_schedule)

model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy']) # make sure we have categorical_crossentropy as loss
model.summary()

#Early stopping
es_callback = EarlyStopping(monitor='val_loss', patience=3)

#fit call to use the datagen. Used 300 epochs
diagramm = model.fit(datagen.flow(X_train,y_train, batch_size = 35) ,epochs = 300 , validation_data = (X_test, y_test))

#Evaluation part; printing accuracy
y_pred_model1 = np.argmax(model.predict(X_test), axis=-1)
y_test_model1 = np.argmax(y_test, axis=-1)
y_train_model1 = np.argmax(y_train, axis=-1)
print('Train Accuracy of th9e model is', accuracy_score(y_train_model1, np.argmax(model.predict(X_train), axis=-1)))
print('Test Accuracy of the model is', accuracy_score(y_test_model1, y_pred_model1))


# Plot accuracy graph
plt.plot(diagramm.history['accuracy'])
plt.plot(diagramm.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()

# Plot loss graph
plt.plot(diagramm.history['loss'])
plt.plot(diagramm.history['val_loss'])
plt.title('model accuracy')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()

"""**Selfmade Model 3**

Best Validation Accu
"""

# Define our CNN
model = Sequential()
model.add(Conv2D(32, (3,3), strides=1, padding='same', activation='relu', input_shape=X_train.shape[1:],kernel_regularizer='l2')) # this layer has 32 filters. The filter size is: (3,3)
model.add(BatchNormalization()) # This is optional to avoid overfitting
model.add(MaxPool2D((2,2), strides=2, padding='same'))

model.add(Conv2D(64 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.1)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))

model.add(Conv2D(128 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.1)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))

model.add(Conv2D(40 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.1)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))


model.add(Flatten())
model.add(Dense(32, activation='relu'))

# Use softmax and 3 ouput layers because of three labels
model.add(Dense(3, activation='softmax')) 

#apply learning rate to adam optimizer
opt = optimizers.Adam(learning_rate=0.0001)
#apply decay rate
lr_schedule = optimizers.schedules.ExponentialDecay(
    initial_learning_rate=1e-5,
    decay_steps=10000,
    decay_rate=0.9)
optimizer = optimizers.Adam(learning_rate=lr_schedule)

model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy']) # make sure we have categorical_crossentropy as loss
model.summary()

#Early stopping (Not used)
es_callback = EarlyStopping(monitor='val_loss', patience=3)

#fit call to use the datagen. Used 300 epochs
diagramm = model.fit(datagen.flow(X_train,y_train, batch_size = 35) ,epochs = 300 , validation_data = (X_test, y_test))

#Evaluation part; printing accuracy
y_pred_model1 = np.argmax(model.predict(X_test), axis=-1)
y_test_model1 = np.argmax(y_test, axis=-1)
y_train_model1 = np.argmax(y_train, axis=-1)
print('Train Accuracy of th9e model is', accuracy_score(y_train_model1, np.argmax(model.predict(X_train), axis=-1)))
print('Test Accuracy of the model is', accuracy_score(y_test_model1, y_pred_model1))


# Plot accuracy graph
plt.plot(diagramm.history['accuracy'])
plt.plot(diagramm.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()

# Plot loss graph
plt.plot(diagramm.history['loss'])
plt.plot(diagramm.history['val_loss'])
plt.title('model accuracy')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()

"""**Selfmade Model 4**

Very Unstable/ Reaches test Accu
"""

model = Sequential()
model.add(Conv2D(32, (3,3), strides=1, padding='same', activation='relu', input_shape=X_train.shape[1:],kernel_regularizer='l2')) # this layer has 32 filters. The filter size is: (3,3)
model.add(BatchNormalization()) # This is optional to avoid overfitting
model.add(MaxPool2D((2,2), strides=2, padding='same'))

model.add(Conv2D(64 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.1)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))

model.add(Conv2D(256 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.1)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))

model.add(Conv2D(64 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.1)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional
model.add(MaxPool2D((3,3) , strides = 2 , padding = 'same'))




model.add(Flatten())
model.add(Dense(32, activation='relu'))
model.add(Dropout(0.1)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional

# Use softmax and 3 ouput layers because of three labels
model.add(Dense(3, activation='softmax')) 

#apply learning rate
opt = optimizers.Adam(learning_rate=0.0001)
lr_schedule = optimizers.schedules.ExponentialDecay(
    initial_learning_rate=0.0001,
    decay_steps=10000,
    decay_rate=0.9)
optimizer = optimizers.Adam(learning_rate=lr_schedule)

model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) # make sure we have categorical_crossentropy as loss
model.summary()

#Early stopping
es_callback = EarlyStopping(monitor='val_loss', patience=3)

#fit call to use the datagen. 100 epichs
diagramm = model.fit(datagen.flow(X_train,y_train, batch_size = 35) ,epochs = 100 , validation_data = (X_test, y_test))

#Evaluation part; printing accuracy
y_pred_model1 = np.argmax(model.predict(X_test), axis=-1)
y_test_model1 = np.argmax(y_test, axis=-1)
y_train_model1 = np.argmax(y_train, axis=-1)
print('Train Accuracy of th9e model is', accuracy_score(y_train_model1, np.argmax(model.predict(X_train), axis=-1)))
print('Test Accuracy of the model is', accuracy_score(y_test_model1, y_pred_model1))


# Plot accuracy graph
plt.plot(diagramm.history['accuracy'])
plt.plot(diagramm.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()

# Plot loss graph
plt.plot(diagramm.history['loss'])
plt.plot(diagramm.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()

"""**Selfmade Model 5**"""

# Define our CNN
model = Sequential()
model.add(Conv2D(32, (3,3), strides=1, padding='same', activation='relu', input_shape=X_train.shape[1:],kernel_regularizer='l2')) # this layer has 32 filters. The filter size is: (3,3)
model.add(BatchNormalization()) # This is optional to avoid overfitting
model.add(MaxPool2D((2,2), strides=2, padding='same'))

model.add(Conv2D(64 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.1)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))

model.add(Conv2D(128 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.1)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))

model.add(Conv2D(40 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.1)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))


model.add(Flatten())
model.add(Dense(32, activation='relu'))

# Use softmax and 3 ouput layers because of three labels
model.add(Dense(3, activation='softmax')) 

#apply learning rate
opt = optimizers.Adam(learning_rate=0.0001)
lr_schedule = optimizers.schedules.ExponentialDecay(
    initial_learning_rate=1e-5,
    decay_steps=10000,
    decay_rate=0.9)
optimizer = optimizers.Adam(learning_rate=lr_schedule)

model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy']) # make sure we have categorical_crossentropy as loss
model.summary()

#Early stopping
es_callback = EarlyStopping(monitor='val_loss', patience=3)

#fit call to use the datagen. 300 epochs
diagramm = model.fit(datagen.flow(X_train,y_train, batch_size = 35) ,epochs = 300 , validation_data = (X_test, y_test))

#Evaluation part; printing accuracy
y_pred_model1 = np.argmax(model.predict(X_test), axis=-1)
y_test_model1 = np.argmax(y_test, axis=-1)
y_train_model1 = np.argmax(y_train, axis=-1)
print('Train Accuracy of th9e model is', accuracy_score(y_train_model1, np.argmax(model.predict(X_train), axis=-1)))
print('Test Accuracy of the model is', accuracy_score(y_test_model1, y_pred_model1))


# Plot accuracy graph
plt.plot(diagramm.history['accuracy'])
plt.plot(diagramm.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()

# Plot loss graph
plt.plot(diagramm.history['loss'])
plt.plot(diagramm.history['val_loss'])
plt.title('model accuracy')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()

"""**Selfmade Model 6**

Higher Dropout Rate

More stable but lower test accuracy
"""

# Define our CNN
model = Sequential()
model.add(Conv2D(32, (3,3), strides=1, padding='same', activation='relu', input_shape=X_train.shape[1:],kernel_regularizer='l2')) # this layer has 32 filters. The filter size is: (3,3)
model.add(BatchNormalization()) # This is optional to avoid overfitting
model.add(Dropout(0.3)) # Dropout is optional.
model.add(MaxPool2D((2,2), strides=2, padding='same'))

model.add(Conv2D(64 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.4)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))

model.add(Conv2D(128 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.4)) # Dropout is optional.
model.add(Dropout(0.3)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))

model.add(Conv2D(40 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.3)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))


model.add(Flatten())
model.add(Dense(32, activation='relu'))
model.add(Dropout(0.2)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional

# Use softmax and 3 ouput layers because of three labels
model.add(Dense(3, activation='softmax'))

#apply learning rate
opt = optimizers.Adam(learning_rate=0.0001)
lr_schedule = optimizers.schedules.ExponentialDecay(
    initial_learning_rate=0.0001,
    decay_steps=10000,
    decay_rate=0.9)
optimizer = optimizers.Adam(learning_rate=lr_schedule)

model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy']) # make sure we have categorical_crossentropy as loss
model.summary()

#Early stopping
es_callback = EarlyStopping(monitor='val_loss', patience=3)


#fit call to use the datagen. 200 epichs
diagramm = model.fit(datagen.flow(X_train,y_train, batch_size = bs) ,epochs = 200 , validation_data = (X_test, y_test))

#Evaluation part; printing accuracy
y_pred_model1 = np.argmax(model.predict(X_test), axis=-1)
y_test_model1 = np.argmax(y_test, axis=-1)
y_train_model1 = np.argmax(y_train, axis=-1)
print('Train Accuracy of the model is', accuracy_score(y_train_model1, np.argmax(model.predict(X_train), axis=-1)))
print('Test Accuracy of the model is', accuracy_score(y_test_model1, y_pred_model1))
print(bs)



# Plot loss graph
plt.plot(diagramm.history['loss'])
plt.plot(diagramm.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()
plt.savefig('loss'+str(bs)+'.png')

# Plot accuracy graph
plt.plot(diagramm.history['accuracy'])
plt.plot(diagramm.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()
plt.savefig('accuracy'+str(bs)+'.png')

"""**Selfmade Model 7**

"""

# Define our CNN
model = Sequential()
model.add(Conv2D(32, (3,3), strides=1, padding='same', activation='relu', input_shape=X_train.shape[1:],kernel_regularizer='l2')) # this layer has 32 filters. The filter size is: (3,3)
model.add(BatchNormalization()) # This is optional to avoid overfitting
model.add(Dropout(0.5)) # Dropout is optional.
model.add(MaxPool2D((2,2), strides=2, padding='same'))

model.add(Conv2D(64 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.5)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))

model.add(Conv2D(128 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.1)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))

model.add(Conv2D(128 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.6)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))

model.add(Conv2D(256 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.1)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))

model.add(Conv2D(256 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.6)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))




model.add(Conv2D(64 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.5)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))


model.add(Flatten())
model.add(Dense(32, activation='relu'))
model.add(Dropout(0.1)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional

# Use softmax and 3 ouput layers because of three labels
model.add(Dense(3, activation='softmax')) 

#apply learning rate
opt = optimizers.Adam(learning_rate=0.0001)
lr_schedule = optimizers.schedules.ExponentialDecay(
    initial_learning_rate=0.0001,
    decay_steps=10000,
    decay_rate=0.9)
optimizer = optimizers.Adam(learning_rate=lr_schedule)

model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) # make sure we have categorical_crossentropy as loss
model.summary()

#Early stopping
es_callback = EarlyStopping(monitor='val_loss', patience=3)


#fit call to use the datagen. 50 epichs
diagramm = model.fit(datagen.flow(X_train,y_train, batch_size = 32) ,epochs = 50 , validation_data = (X_test, y_test))

#Evaluation part; printing accuracy
y_pred_model1 = np.argmax(model.predict(X_test), axis=-1)
y_test_model1 = np.argmax(y_test, axis=-1)
y_train_model1 = np.argmax(y_train, axis=-1)
print('Train Accuracy of the model is', accuracy_score(y_train_model1, np.argmax(model.predict(X_train), axis=-1)))
print('Test Accuracy of the model is', accuracy_score(y_test_model1, y_pred_model1))



# Plot loss graph
plt.plot(diagramm.history['loss'])
plt.plot(diagramm.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()


# Plot accuracy graph
plt.plot(diagramm.history['accuracy'])
plt.plot(diagramm.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()
plt.xlim([0, 1])

"""**Selfmade Model 8**

Small Network high Dropout
"""

# Define our CNN
model = Sequential()
model.add(Conv2D(16, (3,3), strides=1, padding='same', activation='relu', input_shape=X_train.shape[1:],kernel_regularizer='l2')) # this layer has 32 filters. The filter size is: (3,3)
model.add(BatchNormalization()) # This is optional to avoid overfitting
model.add(Dropout(0.1)) # Dropout is optional.
model.add(MaxPool2D((2,2), strides=2, padding='same'))


model.add(Conv2D(32 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.1)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))

model.add(Conv2D(32 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.5)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))



model.add(Flatten())
model.add(Dense(16, activation='relu'))
model.add(Dropout(0.1)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional

# Use softmax and 3 ouput layers because of three labels
model.add(Dense(3, activation='softmax')) 

#apply learning rate
opt = optimizers.Adam(learning_rate=0.00005)
lr_schedule = optimizers.schedules.ExponentialDecay(
    initial_learning_rate=0.0001,
    decay_steps=10000,
    decay_rate=0.9)
optimizer = optimizers.Adam(learning_rate=lr_schedule)

model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) # make sure we have categorical_crossentropy as loss
model.summary()

#Early stopping
es_callback = EarlyStopping(monitor='val_loss', patience=3)


#fit call to use the datagen. 100 epichs
diagramm = model.fit(datagen.flow(X_train,y_train, batch_size = 32) ,epochs = 100 , validation_data = (X_test, y_test))

#Evaluation part; printing accuracy
y_pred_model1 = np.argmax(model.predict(X_test), axis=-1)
y_test_model1 = np.argmax(y_test, axis=-1)
y_train_model1 = np.argmax(y_train, axis=-1)
print('Train Accuracy of the model is', accuracy_score(y_train_model1, np.argmax(model.predict(X_train), axis=-1)))
print('Test Accuracy of the model is', accuracy_score(y_test_model1, y_pred_model1))



# Plot loss graph
plt.plot(diagramm.history['loss'])
plt.plot(diagramm.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()


# Plot accuracy graph
plt.plot(diagramm.history['accuracy'])
plt.plot(diagramm.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()

"""**Selfmade Model 9**

Smal Network
"""

# Define our CNN
model = Sequential()
model.add(Conv2D(16, (3,3), strides=1, padding='same', activation='relu', input_shape=X_train.shape[1:],kernel_regularizer='l2')) # this layer has 32 filters. The filter size is: (3,3)
model.add(BatchNormalization()) # This is optional to avoid overfitting
model.add(Dropout(0.1)) # Dropout is optional.
model.add(MaxPool2D((2,2), strides=2, padding='same'))


model.add(Conv2D(32 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.1)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))

model.add(Conv2D(32 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))
model.add(Dropout(0.5)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional
model.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))



model.add(Flatten())
model.add(Dense(16, activation='relu'))
model.add(Dropout(0.1)) # Dropout is optional.
model.add(BatchNormalization()) # BatchNormalization is optional

# Use softmax and 3 ouput layers because of three labels
model.add(Dense(3, activation='softmax')) 

#apply learning rate
opt = optimizers.Adam(learning_rate=0.00005)
lr_schedule = optimizers.schedules.ExponentialDecay(
    initial_learning_rate=0.0001,
    decay_steps=10000,
    decay_rate=0.9)
optimizer = optimizers.Adam(learning_rate=lr_schedule)

model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) # make sure we have categorical_crossentropy as loss
model.summary()

#Early stopping
es_callback = EarlyStopping(monitor='val_loss', patience=3)


#fit call to use the datagen. 500 epichs
diagramm = model.fit(datagen.flow(X_train,y_train, batch_size = 32) ,epochs = 500 , validation_data = (X_test, y_test))

#Evaluation part; printing accuracy
y_pred_model1 = np.argmax(model.predict(X_test), axis=-1)
y_test_model1 = np.argmax(y_test, axis=-1)
y_train_model1 = np.argmax(y_train, axis=-1)
print('Train Accuracy of the model is', accuracy_score(y_train_model1, np.argmax(model.predict(X_train), axis=-1)))
print('Test Accuracy of the model is', accuracy_score(y_test_model1, y_pred_model1))



# Plot loss graph
plt.plot(diagramm.history['loss'])
plt.plot(diagramm.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()


# Plot accuracy graph
plt.plot(diagramm.history['accuracy'])
plt.plot(diagramm.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()

"""# **Application Model start here**

**VGG16**
"""

# load existing CNN from tensorflow. Here we use VGG16, pretrained on imagenet. we will not include_top.
# include_top means the flatten layer + the following dense layers. Wen only use the CNN-blocks from this network.
model = applications.VGG16(weights='imagenet', include_top=False, input_shape=X_train.shape[1:])


flat1 = Flatten()(model.output) # add a flatten to the VGG16
class1 = Dense(256, activation='relu')(flat1) # add a Dense layer after the flatten
# Use softmax and 3 ouput layers because of three labels
output = Dense(3, activation='softmax')(class1)

model = Model(inputs=model.inputs, outputs=output) # define our final model. The first part is from VGG16 and the output is the layer defined above

#apply learning rate
opt = optimizers.Adam(learning_rate=0.00005)
lr_schedule = optimizers.schedules.ExponentialDecay(
    initial_learning_rate=1e-5,
    decay_steps=10000,
    decay_rate=0.9)
optimizer = optimizers.Adam(learning_rate=lr_schedule)

model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) # make sure we have categorical_crossentropy as loss
model.summary()

#Early stopping
es_callback = EarlyStopping(monitor='val_loss', patience=3)

#fit call to use the datagen. 30 epichs
diagramm = model.fit(datagen.flow(X_train,y_train, batch_size = 35) ,epochs = 30 , validation_data = (X_test, y_test))

#Evaluation part; printing accuracy
y_pred_model1 = np.argmax(model.predict(X_test), axis=-1)
y_test_model1 = np.argmax(y_test, axis=-1)
y_train_model1 = np.argmax(y_train, axis=-1)
print('Train Accuracy of the model is', accuracy_score(y_train_model1, np.argmax(model.predict(X_train), axis=-1)))
print('Test Accuracy of the model is', accuracy_score(y_test_model1, y_pred_model1))


# Plot accuracy graph
plt.plot(diagramm.history['accuracy'])
plt.plot(diagramm.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()

# Plot loss graph
plt.plot(diagramm.history['loss'])
plt.plot(diagramm.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()

"""**ResNet152V2**"""

# load existing CNN from tensorflow. Here we use VGG16, pretrained on imagenet. we will not include_top.
# include_top means the flatten layer + the following dense layers. Wen only use the CNN-blocks from this network.
model = applications.ResNet152V2(weights='imagenet', include_top=False, input_shape=X_train.shape[1:] ,kernel_regularizer='l2')

flat1 = Flatten()(model.output) # add a flatten to the VGG16
class1 = Dense(256, activation='relu')(flat1) # add a Dense layer after the flatten
# Use softmax and 3 ouput layers because of three labels
output = Dense(3, activation='softmax')(class1) # add a dense layer after the 256-dense layer

model = Model(inputs=model.inputs, outputs=output) # define our final model. The first part is from VGG16 and the output is the layer defined above

model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
model.summary()


model.fit(datagen.flow(X_train,y_train, batch_size = 32) ,epochs = 10 , validation_data = (X_test, y_test))
y_pred_model1 = np.argmax(model.predict(X_test), axis=-1)
y_test_model1 = np.argmax(y_test, axis=-1)
y_train_model1 = np.argmax(y_train, axis=-1)
print("Resnet152")
print('Train Accuracy of the model is', accuracy_score(y_train_model1, np.argmax(model.predict(X_train), axis=-1)))
print('Test Accuracy of the model is', accuracy_score(y_test_model1, y_pred_model1))

# Plot accuracy graph
plt.plot(diagramm.history['accuracy'])
plt.plot(diagramm.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()

# Plot loss graph
plt.plot(diagramm.history['loss'])
plt.plot(diagramm.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()

"""**EfficientNetB7**"""

# load existing CNN from tensorflow. Here we use VGG16, pretrained on imagenet. we will not include_top.
# include_top means the flatten layer + the following dense layers. Wen only use the CNN-blocks from this network.
model = applications.EfficientNetB7(weights='imagenet', include_top=False, input_shape=X_train.shape[1:])

flat1 = Flatten()(model.output) # add a flatten to the VGG16
class1 = Dense(256, activation='relu')(flat1) # add a Dense layer after the flatten
class2 = BatchNormalization()(class1) # This is optional to avoid overfitting
class3 = Dropout(0.2)(class2) # Dropout is optional.


# Use softmax and 3 ouput layers because of three labels
output = Dense(3, activation='softmax')(class3) # add a dense layer after the 256-dense layer

model = Model(inputs=model.inputs, outputs=output) # define our final model. The first part is from VGG16 and the output is the layer defined above

model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
model.summary()


model.fit(datagen.flow(X_train,y_train, batch_size = 32) ,epochs = 30 , validation_data = (X_test, y_test))
y_pred_model1 = np.argmax(model.predict(X_test), axis=-1)
y_test_model1 = np.argmax(y_test, axis=-1)
y_train_model1 = np.argmax(y_train, axis=-1)
print("Inception EfficientNetB7")
print('Train Accuracy of the model is', accuracy_score(y_train_model1, np.argmax(model.predict(X_train), axis=-1)))
print('Test Accuracy of the model is', accuracy_score(y_test_model1, y_pred_model1))

# Plot accuracy graph
plt.plot(diagramm.history['accuracy'])
plt.plot(diagramm.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.xlim([0, 1])
plt.show()

# Plot loss graph
plt.plot(diagramm.history['loss'])
plt.plot(diagramm.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()

"""**ResNet50**"""

# load existing CNN from tensorflow. Here we use VGG16, pretrained on imagenet. we will not include_top.
# include_top means the flatten layer + the following dense layers. Wen only use the CNN-blocks from this network.
model = applications.ResNet50(weights='imagenet', include_top=False, input_shape=X_train.shape[1:])

flat1 = Flatten()(model.output) # add a flatten to the VGG16
class1 = Dense(256, activation='relu')(flat1) # add a Dense layer after the flatten
# Use softmax and 3 ouput layers because of three labels
output = Dense(3, activation='softmax')(class1) # add a dense layer after the 256-dense layer

model = Model(inputs=model.inputs, outputs=output) # define our final model. The first part is from VGG16 and the output is the layer defined above


#apply learning rate
opt = optimizers.Adam(learning_rate=0.00005)
lr_schedule = optimizers.schedules.ExponentialDecay(
    initial_learning_rate=1e-5,
    decay_steps=10000,
    decay_rate=0.9)
optimizer = optimizers.Adam(learning_rate=lr_schedule)

model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) # make sure we have categorical_crossentropy as loss
model.summary()


model.fit(datagen.flow(X_train,y_train, batch_size = 32) ,epochs = 10 , validation_data = (X_test, y_test))
y_pred_model1 = np.argmax(model.predict(X_test), axis=-1)
y_test_model1 = np.argmax(y_test, axis=-1)
y_train_model1 = np.argmax(y_train, axis=-1)
print("Inception Resnet 50")
print('Train Accuracy of the model is', accuracy_score(y_train_model1, np.argmax(model.predict(X_train), axis=-1)))
print('Test Accuracy of the model is', accuracy_score(y_test_model1, y_pred_model1))

"""**MobilNet**"""

# load existing CNN from tensorflow. Here we use VGG16, pretrained on imagenet. we will not include_top.
# include_top means the flatten layer + the following dense layers. Wen only use the CNN-blocks from this network.
model = applications.MobileNet(weights='imagenet', include_top=False, input_shape=X_train.shape[1:])

flat1 = Flatten()(model.output) # add a flatten to the VGG16
class1 = Dense(128, activation='relu')(flat1) # add a Dense layer after the flatten
#class2 = add(Dropout(0.1)) # Dropout is optional.
class2 = BatchNormalization()(class1) # BatchNormalization is optional

# Use softmax and 3 ouput layers because of three labels
output = Dense(3, activation='softmax')(class1) # add a dense layer after the 256-dense layer

model = Model(inputs=model.inputs, outputs=output) # define our final model. The first part is from VGG16 and the output is the layer defined above


lr_schedule = optimizers.schedules.ExponentialDecay(
    initial_learning_rate=0.00001,
    decay_steps=10000,
    decay_rate=0.9)
optimizer = optimizers.Adam(learning_rate=0.0005)
model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])
model.summary()


diagramm =model.fit(datagen.flow(X_train,y_train, batch_size = 32) ,epochs = 20 , validation_data = (X_test, y_test))
y_pred_model1 = np.argmax(model.predict(X_test), axis=-1)
y_test_model1 = np.argmax(y_test, axis=-1)
y_train_model1 = np.argmax(y_train, axis=-1)
print("Inception MobileNet")
print('Train Accuracy of the model is', accuracy_score(y_train_model1, np.argmax(model.predict(X_train), axis=-1)))
print('Test Accuracy of the model is', accuracy_score(y_test_model1, y_pred_model1))

# Plot accuracy graph
plt.plot(diagramm.history['accuracy'])
plt.plot(diagramm.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()

# Plot loss graph
plt.plot(diagramm.history['loss'])
plt.plot(diagramm.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()

"""**EfficientNetB0**"""

# load existing CNN from tensorflow. Here we use VGG16, pretrained on imagenet. we will not include_top.
# include_top means the flatten layer + the following dense layers. Wen only use the CNN-blocks from this network.
model = applications.EfficientNetB0(weights='imagenet', include_top=False, input_shape=X_train.shape[1:])

flat1 = Flatten()(model.output) # add a flatten to the VGG16
class1 = Dense(256, activation='relu')(flat1) # add a Dense layer after the flatten
# Use softmax and 3 ouput layers because of three labels
output = Dense(3, activation='softmax')(class1) # add a dense layer after the 256-dense layer

model = Model(inputs=model.inputs, outputs=output) # define our final model. The first part is from VGG16 and the output is the layer defined above


lr_schedule = optimizers.schedules.ExponentialDecay(
    initial_learning_rate=1e-5,
    decay_steps=10000,
    decay_rate=0.9)
optimizer = optimizers.Adam(learning_rate=lr_schedule)
model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])
model.summary()


diagramm = model.fit(datagen.flow(X_train,y_train, batch_size = 32) ,epochs = 100 , validation_data = (X_test, y_test))
y_pred_model1 = np.argmax(model.predict(X_test), axis=-1)
y_test_model1 = np.argmax(y_test, axis=-1)
y_train_model1 = np.argmax(y_train, axis=-1)
print("Inception Resnet 50")
print('Train Accuracy of the model is', accuracy_score(y_train_model1, np.argmax(model.predict(X_train), axis=-1)))
print('Test Accuracy of the model is', accuracy_score(y_test_model1, y_pred_model1))

# Plot accuracy graph
plt.plot(diagramm.history['accuracy'])
plt.plot(diagramm.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()

# Plot loss graph
plt.plot(diagramm.history['loss'])
plt.plot(diagramm.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()

"""**DenseNet201**"""

# load existing CNN from tensorflow. Here we use VGG16, pretrained on imagenet. we will not include_top.
# include_top means the flatten layer + the following dense layers. Wen only use the CNN-blocks from this network.
model = applications.DenseNet201(weights='imagenet', include_top=False, input_shape=X_train.shape[1:])

flat1 = Flatten()(model.output) # add a flatten to the VGG16
class1 = Dense(256, activation='relu')(flat1) # add a Dense layer after the flatten
class2 = BatchNormalization()(class1) # This is optional to avoid overfitting
class3 = Dropout(0.2)(class2) # Dropout is optional.

# Use softmax and 3 ouput layers because of three labels
output = Dense(3, activation='softmax')(class3) # add a dense layer after the 256-dense layer

model = Model(inputs=model.inputs, outputs=output) # define our final model. The first part is from VGG16 and the output is the layer defined above


lr_schedule = optimizers.schedules.ExponentialDecay(
    initial_learning_rate=0.00001,
    decay_steps=10000,
    decay_rate=0.9)
optimizer = optimizers.Adam(learning_rate=lr_schedule)
model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])
model.summary()



diagramm = model.fit(datagen.flow(X_train,y_train, batch_size = 32) ,epochs = 100 , validation_data = (X_test, y_test))
y_pred_model1 = np.argmax(model.predict(X_test), axis=-1)
y_test_model1 = np.argmax(y_test, axis=-1)
y_train_model1 = np.argmax(y_train, axis=-1)
print("Inception DensNet201")
print('Train Accuracy of the model is', accuracy_score(y_train_model1, np.argmax(model.predict(X_train), axis=-1)))
print('Test Accuracy of the model is', accuracy_score(y_test_model1, y_pred_model1))

# Plot accuracy graph
plt.plot(diagramm.history['accuracy'])
plt.plot(diagramm.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.xlim([0, 1])
plt.show()

# Plot loss graph
plt.plot(diagramm.history['loss'])
plt.plot(diagramm.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()