-
Notifications
You must be signed in to change notification settings - Fork 3
/
TestModelAUC.py
98 lines (77 loc) · 2.89 KB
/
TestModelAUC.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
import torch
from Train import MyDataset
from imports.ParametersManager import *
from Mantra_Net import *
from matplotlib import pyplot as plt
import torchvision.transforms as transforms
'''
This python file is used to calculate the model' s ROC and AUC value of your trained model.
'''
# Enter the *.pt model file name here to load parameters
DIR = './Pre_TrainedModel/'
# ===You need to change the name of the model here =====
ModelName = DIR + 'MantraNet on NIST16_model (8).pt'
# ====================================================
parManager = ParametersManager('cuda')
parManager.loadFromFile(ModelName)
print("This model has done : {} Epochs.".format(parManager.EpochDone))
model = ManTraNet()
model.cuda()
parManager.setModelParameters(model)
# ===========hyper parameters=============
dataSetChoosen = 'Train'
resolution = 100
# ========================================
dataDIR ={
'Whole' : './NIST2016/index.csv',
'Train' : './NIST2016/Train.csv',
'Test' : './NIST2016/Test.csv'
}
data = MyDataset(dataDIR[dataSetChoosen])
with torch.no_grad():
model.eval()
Loader = DataLoader(data, pin_memory=True, batch_size=1)
step = int(len(Loader) / 100)
print(step)
trans = transforms.ToPILImage()
labels = []
prediction = []
for id, (x,label) in enumerate(Loader):
labels.append(torch.squeeze(torch.squeeze(label , dim=0), dim=0) )
out = model(x.cuda())
prediction.append(torch.squeeze(torch.squeeze(out.cpu(), dim=0), dim=0) )
if id % step == 0:
print('{:.2f}%'.format(id/len(Loader) * 100))
labels = torch.stack(labels, dim = 0)
prediction = torch.stack(prediction, dim=0)
print(labels.shape)
print(prediction.shape)
def cal_ROC_rate(labels, predict:torch.Tensor, threshold:float):
mask = (predict > threshold).float()
TP, TN, FP, FN = 0, 0, 0, 0
TP += torch.sum((mask == 1) & (labels == 1))
TN += torch.sum((mask == 0) & (labels == 0))
FP += torch.sum((mask == 1) & (labels == 0))
FN += torch.sum((mask == 0) & (labels == 1))
TPR = TP / (TP + FN) # True positive rate
FPR = FP / (TN + FP) # False Positive Rate
return TPR, FPR
TPR = []
FPR = []
for threshold in range(resolution):
threshold /= resolution
# print(threshold)
t_TPR, t_FPR = cal_ROC_rate(labels, prediction, threshold)
TPR.append(t_TPR.cpu())
FPR.append(t_FPR.cpu())
TPR_array = sorted(TPR)
FPR_array = sorted(FPR)
AUC = np.trapz(TPR_array, FPR_array)
plt.xlabel('FPR')
plt.ylabel('TPR')
plt.xlim((0,1))
plt.ylim((0,1))
plt.title('ROC of {} Epoch ManTra-Net trainning on NIST16 {} dataset'.format(parManager.EpochDone, dataSetChoosen))
plt.text(0.7, 0.3,r'$AUC$:{:.6F}'.format(AUC))
plt.plot(FPR, TPR) # front parameter is for x, back parameter is for y
plt.show()