- issue #5 :

	- Added measure_performance for base evaluation
	- Fixed results reproting:
		- Accuracy, AUC, FPR, TPR are returned first as dict then transformed to arrays
- issue #2 :
	- Added EarlyStopping callback
	- Added confution matrix to metrics, FPR and TPR
 On branch dev
 Changes to be committed:
	modified:   aawedha/evaluation/base.py
	modified:   aawedha/evaluation/cross_subject.py
	modified:   aawedha/evaluation/single_subject.py
	modified:   aawedha/scripts/single_subj.py

aawedha/evaluation/base.py

@@ -3,6 +3,7 @@
 
 '''
 from tensorflow.keras.utils import to_categorical
+from sklearn.metrics import roc_curve, auc, confusion_matrix
 import numpy as np
 import random
 import abc
@@ -20,6 +21,7 @@ def __init__(self, n_subjects=0, partition=[],
  self.folds = folds
  self.dataset = dataset
  self.model = model
+ self.cm = [] # confusion matrix per fold
  self.results = {} # dict 
  self.model_history = {}
  self.verbose = verbose
@@ -37,22 +39,54 @@ def run_evaluation(self):
  '''
  pass
 
- def results_reports(self, res):
+
+ def measure_performance(self, Y_test, probs):
+ '''
+ '''
+ preds = probs.argmax(axis=-1)
+ y_true = Y_test.argmax(axis=-1)
+ classes = Y_test.shape[1]
+ acc = np.mean(preds == y_true) 
+
+ self.cm.append(confusion_matrix(Y_test.argmax(axis=-1), preds))
+
+ if classes == 2: 
+ fp_rate, tp_rate, thresholds = roc_curve(y_true, probs[:,1])
+ auc_score = auc(fp_rate, tp_rate)
+ return acc.item(), auc_score.item(), fp_rate, tp_rate
+ else:
+ return acc.item()
+
+
+
+ def results_reports(self, res, tfpr={}):
  '''
  '''
  results = {}
 
- if res.ndim == 3:
+ if tfpr:
  # res : (metric, subjects, folds)
- means = res.mean(axis=-1) # mean across folds
- results['acc'] = means[0]
- results['acc_mean'] = means[0].mean()
- results['auc'] = means[1]
- results['auc_mean'] = means[1].mean()
+ # means = res.mean(axis=-1) # mean across folds
+ r1 = np.array(res['acc'])
+ results['acc'] = r1
+ results['acc_mean_per_fold'] = r1.mean(axis=0)
+ results['acc_mean_per_subj'] = r1.mean(axis=1)
+ results['acc_mean'] = r1.mean()
+
+ r2 = np.array(res['auc'])
+ results['auc'] = r2
+ results['auc_mean_per_fold'] = r2.mean(axis=0)
+ results['auc_mean_per_subj'] = r2.mean(axis=1)
+ results['auc_mean'] = r2.mean()
+
+ results['fpr'] = tfpr['fp']
+ results['tpr'] = tfpr['tp']
  else:
  # res : (subjects, folds)
- results['acc'] = res.mean(axis=-1) # mean across folds
- results['acc_mean'] = res.mean() # mean across subjects
+ results['acc'] = res
+ results['acc_mean_per_fold'] = res.mean(axis=0) # mean across folds
+ results['acc_mean_per_subject'] = res.mean(axis=1) # mean across subjects and folds
+ results['acc_mean'] = res.mean()
 
  return results 
 
@@ -119,6 +153,7 @@ def transform_scale(self, X, mu, sigma):
  X = np.divide(X, sigma[None,:,:])
  return X
 
+
  def class_weights(self, y):
  '''
  '''
@@ -147,7 +182,7 @@ def labels_to_categorical(self, y):
  else:
  y = to_categorical(y-1)
  return y
-
+ 
  def save_model(self, folderpath=None):
  '''
  '''

aawedha/evaluation/cross_subject.py

@@ -26,31 +26,37 @@ def generate_split(self, nfolds=30, excl=True):
  self.folds = self.get_folds(nfolds, self.n_subjects, train_phase, val_phase, test_phase, exclude_subj=excl)
 
 
- def run_evaluation(self):
+ def run_evaluation(self, clbs=[]):
  '''
  '''
  # generate folds if folds are empty
  if not self.folds:
  self.folds = self.generate_split(nfolds=30)
  # 
- res_acc = []
- res_auc = []
+ res_acc , res_auc = [], []
+ res_tp, res_fp = [], []
 
  for fold in range(len(self.folds)):
- acc_per_fold, auc_per_fold = self._cross_subject(fold)
- res_acc.append(acc_per_fold)
- if auc_per_fold:
- res_auc.append(auc_per_fold) 
+ rets = self._cross_subject(fold,clbs)
+ if type(rets) is tuple:
+ res_acc.append(rets[0])
+ res_auc.append(rets[1]) 
+ res_fp.append(rets[2])
+ res_tp.append(rets[3])
+ else:
+ res_acc.append(rets) 
 
  # Aggregate results
  if res_auc:
  res = np.array([res_acc, res_auc])
+ tfpr = np.array([res_fp, res_tp])
  else:
- res = np.array(res_acc) 
+ res = np.array(res_acc)
+ tfpr = [] 
 
- self.results = self.results_reports(res) 
+ self.results = self.results_reports(res, tfpr) 
 
- def _cross_subject(self, fold):
+ def _cross_subject(self, fold, clbs=[]):
  '''
  '''
  # 
@@ -71,18 +77,12 @@ def _cross_subject(self, fold):
 
  self.model_history = self.model.fit(X_train, Y_train, batch_size = 96, epochs = 500, 
  verbose = self.verbose, validation_data=(X_val, Y_val),
- class_weight = cws)
+ class_weight = cws, callbacks=clbs)
  # train/val
  probs = self.model.predict(X_test)
- preds = probs.argmax(axis = -1) 
- acc = np.mean(preds == Y_test.argmax(axis=-1))
-
- res_acc.append(acc)
- if classes.size == 2:
- auc_score = roc_auc_score(Y_test.argmax(axis=-1), preds)
- res_auc.append(auc_score) 
-
- return res_acc, res_auc
+ rets = self.measure_performance(Y_test, probs) 
+
+ return rets
 
  def _split_set(self, fold):
  '''

aawedha/evaluation/single_subject.py

@@ -1,6 +1,8 @@
 from aawedha.evaluation.base import Evaluation
 from sklearn.model_selection import KFold
-from sklearn.metrics import roc_auc_score
+# from sklearn.metrics import roc_auc_score
+from sklearn.metrics import roc_curve, auc
+from tensorflow.keras.callbacks import EarlyStopping
 import numpy as np
 import random 
 
@@ -42,15 +44,15 @@ def generate_split(self, nfolds=30, strategy='Kfold'):
  elif strategy == 'Shuffle':
  self.folds = self.get_folds(nfolds, n_trials, train_phase, val_phase, test_phase, exclude_subj=False)
 
- def run_evaluation(self, subject=None):
+ def run_evaluation(self, subject=None, clbs=[]):
  '''
  ''' 
  # generate folds if folds are empty
  if not self.folds:
  self.folds = self.generate_split(nfolds=30)
  # 
  res_acc = []
- res_auc = []
+ res_auc, res_tp, res_fp = [], [], []
 
  independent_test = False
  # equal_subjects = self._get_n_subj()
@@ -72,22 +74,32 @@ def run_evaluation(self, subject=None):
 
  for subj in operations:
  # res_per_subject, avg_res = self._single_subject(subj, independent_test)
- acc_per_subject, auc_per_subject = self._single_subject(subj, independent_test)
- res_acc.append(acc_per_subject)
- if auc_per_subject:
- res_auc.append(auc_per_subject) 
+ # acc_per_subject, auc_per_subject, fp, tp = self._single_subject(subj, independent_test, clbs)
+ rets = self._single_subject(subj, independent_test, clbs)
+ if type(rets[0]) is tuple:
+ res_acc.append([elm[0] for elm in rets])
+ res_auc.append([elm[1] for elm in rets])
+ res_fp.append([elm[2] for elm in rets])
+ res_tp.append([elm[3] for elm in rets]) 
+ else:
+ res_acc.append(rets) 
 
  # Aggregate results
- # res_acc = np.array(res_acc)
- if res_auc:
- res = np.array([res_acc, res_auc])
+
+ tfpr = {}
+ if res_auc: 
+ res = {}
+ res['acc'] = res_acc
+ res['auc'] = res_auc 
+ tfpr['fp'] = res_fp
+ tfpr['tp'] = res_tp
  else:
- res = np.array(res_acc)
+ res = np.array(res_acc) 
 
- self.results = self.results_reports(res)
+ self.results = self.results_reports(res, tfpr)
 
 
- def _single_subject(self, subj, indie=False):
+ def _single_subject(self, subj, indie=False, clbs=[]):
  '''
  '''
  # prepare data
@@ -100,8 +112,9 @@ def _single_subject(self, subj, indie=False):
  # 
  classes = np.unique(y)
  y = self.labels_to_categorical(y)
- res_acc = [] 
- res_auc = []
+ # res_acc = [] 
+ # res_auc = []
+ rets = []
  # get in the fold!!!
  for fold in range(len(self.folds)):
  #
@@ -120,19 +133,14 @@ def _single_subject(self, subj, indie=False):
  # rename the fit method
  self.model_history = self.model.fit(X_train, Y_train, batch_size = 64, epochs = 500, 
  verbose = self.verbose, validation_data=(X_val, Y_val),
- class_weight = class_weights)
- # train/val 
- # test 
+ class_weight = class_weights, callbacks=clbs)
+
  probs = self.model.predict(X_test)
- preds = probs.argmax(axis = -1) 
- acc = np.mean(preds == Y_test.argmax(axis=-1))
- res_acc.append(acc.item())
- if classes.size == 2:
- auc_score = roc_auc_score(Y_test.argmax(axis=-1), preds)
- res_auc.append(auc_score.item()) 
-
+ rets.append(self.measure_performance(Y_test, probs)) 
+
+ return rets 
  # res = [] # shape: (n_folds, 2) 
- return res_acc, res_auc 
+
 
  def _fuse_data(self):
  '''

aawedha/scripts/single_subj.py

@@ -1,11 +1,13 @@
 '''
  - main script for evaluating a model(s) on a dataset (s)
 '''
-from aawedha.io.bci_comp_iv_2a import Comp_IV_2a 
+from aawedha.io.inria_ern import Inria_ERN 
 from aawedha.evaluation.single_subject import SingleSubject
-from aawedha.paradigms.motor_imagery import MotorImagery
+from aawedha.paradigms.erp import ERP
 from aawedha.paradigms.subject import Subject
 from aawedha.models.EEGModels import EEGNet
+from tensorflow.keras.metrics import AUC
+
 import numpy as np
 
 from tensorflow.keras import backend as K
@@ -29,10 +31,9 @@
 # load epoched dataset
 f = 'data/comp_IV_2a/epoched/Comp_IV_2a.pkl'
 
-dt = Comp_IV_2a().load_set(f)
+dt = Inria_ERN().load_set(f)
 subjects, samples, channels, trials = dt.epochs.shape
 n_classes = np.unique(dt.y).size
-prt = [2,1,1]
 
 # subject = 3 # subject number
 
@@ -53,9 +54,10 @@
  )
 # train/test model
 evl.run_evaluation()
-# evl.run_evaluation(subject) # specific subject
-#
+
+print(f'Results : {evl.results}')
+
 # print(f'Subject N: {subject+1} Acc: {evl.results["acc"]} ')
-print(f' Acc: {evl.results["acc"]} ')
+# print(f' Acc: {evl.results["acc"]} ')
 
 # Visualize learning curves