-
Notifications
You must be signed in to change notification settings - Fork 0
/
eval.py
175 lines (150 loc) · 5.92 KB
/
eval.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
############################################################################
## Evaluate a trained model ##
############################################################################
# import libraries
import tensorflow as tf
import numpy as np
import pandas as pd
import os
from sklearn.metrics import confusion_matrix, roc_curve, auc, mean_absolute_error, mean_squared_error, classification_report, accuracy_score
from data_manager import DataManager
from plot_data import PlotData
from keras.models import model_from_json
from keras.optimizers import SGD, RMSprop, Adam
from tensorflow_addons.optimizers import AdamW
from keras import backend as K
K.image_data_format()
# Model type
model_type = 'S-Model'
#model_type = 'TL-Model'
#model_type = 'EN-Model'
# Images width, height & channels
img_height = 512
img_width = 512
num_channels = 3
# for plotting confusion matrix
cm_plot_labels = ['cover', 'stego']
# test dataset
model_test_dataset = 'dataset_' + model_type
# path to saved model files
base_path = 'D:/CNN/StegoCNN'
train_path = base_path + '/trained/' + model_type
saved_model_weights_path = train_path + '/Best-weights.h5'
saved_model_arch_path = train_path + '/model.json'
test_data_dir = 'D:/CNN/dataset/test'
# paths to save outputs
train_log_data_file = train_path + '/model_train.csv'
plt_cm = train_path + '/model_cm.png'
plt_normalized_cm = train_path + '/model_norm_cm.png'
plt_roc = train_path + '/model_roc.png'
plt_accuracy = train_path + '/model_accuracy.png'
plt_loss = train_path + '/model_loss.png'
eval_report = train_path + '/eval_report.txt'
# Loss function
model_loss_function = 'binary_crossentropy'
# Define optimizers
# learning rate
lr=0.001
# momentum
m=0.8
model_optimizer_sgd = SGD(learning_rate=lr, decay=1e-6, momentum=m, nesterov=True)
model_optimizer_rmsprop = RMSprop(learning_rate=lr)
model_optimizer_adam = Adam(learning_rate=lr, decay=0.00001)
model_optimizer_adamW = AdamW(learning_rate=lr, weight_decay=0.004)
# For testing of different optimizers
model_optimizer = model_optimizer_sgd
# model metrics to evaluate training
model_metrics = ["accuracy"]
# batch size
batch_size = 16
# Load architecture and weights of the model
def load_model():
print("saved_model_arch_path: " + saved_model_arch_path)
print("saved_model_weights_path:" + saved_model_weights_path)
json_file = open(saved_model_arch_path, 'r')
loaded_model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(loaded_model_json)
# load weights into new model
loaded_model.load_weights(saved_model_weights_path)
return loaded_model
def delete_file(filename):
if os.path.exists(filename):
os.remove(filename)
pass
def main():
# init DataManager class
dataManager = DataManager(img_height, img_width)
# init PlotData class
plotData = PlotData()
# load model
model = load_model()
# get test data
test_data = dataManager.get_test_data(test_data_dir)
# start the evaluation process
print("===================== Start evaluaton =========================")
y_true = test_data.classes
# Confution Matrix and Classification Report
Y_pred = model.predict(test_data)
y_pred = np.argmax(Y_pred, axis=1)
print(y_true)
print(y_pred)
# plot confusion matrix
cm = confusion_matrix(y_true, y_pred)
plotData.plot_confusion_matrix(cm, cm_plot_labels, plt_cm, title='Confusion Matrix')
plotData.plot_confusion_matrix(cm, cm_plot_labels, plt_normalized_cm, normalize=True, title='normalized Confusion Matrix')
# Compute ROC curve and ROC area for each class
roc_auc = plotData.plot_roc(y_true, y_pred, plt_roc)
mae = mean_absolute_error(y_true, y_pred)
mse = mean_squared_error(y_true, y_pred)
accuracy = accuracy_score(y_true, y_pred)
print('Mean Absolute Error (MAE): ' + str(mae))
print('Mean Squared Error (MSE): ' + str(mse))
print('Area Under the Curve (AUC): ' + str(roc_auc))
c_report = classification_report(y_true, y_pred, target_names=cm_plot_labels)
print(c_report)
delete_file(eval_report)
#Save the evaluation report
with open(eval_report, 'a') as f:
f.write('\n\n')
f.write('******************************************************\n')
f.write('** Evaluation Report **\n')
f.write('******************************************************\n')
f.write('\n\n')
f.write('Accuracy Score: ' + str(accuracy))
f.write('\n\n')
f.write('Mean Absolute Error (MAE): ' + str(mae))
f.write('\n\n')
f.write('Mean Squared Error (MSE): ' + str(mse))
f.write('\n\n')
f.write('Area Under the Curve (AUC): ' + str(roc_auc))
f.write('\n\n')
f.write('- Confusion Matrix:\n')
f.write(str(cm))
f.write('\n\n')
f.write('Normalized Confusion Matrix:\n')
cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
f.write(str(cm))
f.write('\n\n')
f.write('Classification report:\n')
f.write(str(c_report))
f.close()
train_validation = ['train', 'validation']
data = pd.read_csv(train_log_data_file)
acc = data['accuracy'].values
val_acc = data['val_accuracy'].values
loss = data['loss'].values
val_loss = data['val_loss'].values
# plot graphs for loss and accuracy
plotData.plot_2d(acc, val_acc, 'epoch', 'accuracy',
'Model Accuracy', train_validation, plt_accuracy)
plotData.plot_2d(loss, val_loss, 'epoch', 'loss',
'Model Loss', train_validation, plt_loss)
# evalute the model
print("==================== compile model ========================")
model.compile(loss=model_loss_function, optimizer=model_optimizer, metrics=model_metrics)
score = model.evaluate(test_data)
print('Test Loss:', score[0])
print('Test Accuracy:', score[1])
if __name__ == "__main__":
main()