From 883c46b06225f53ecd791f1c9fcac0093ed4806b Mon Sep 17 00:00:00 2001
From: Florian Kunneman <f.kunneman@gmail.com>
Date: Thu, 12 Jan 2017 14:53:01 +0100
Subject: [PATCH] enabled ordinal evaluation on class level

---
 evaluation.py       | 37 +++++++++++++++++++++++++++++--------
 tests/evaluation.py | 15 +++++++++++++--
 2 files changed, 42 insertions(+), 10 deletions(-)

diff --git a/evaluation.py b/evaluation.py
index 84f17cda..11b88a28 100644
--- a/evaluation.py
+++ b/evaluation.py
@@ -43,7 +43,6 @@
 import datetime
 import os.path
 
-
 def auc(x, y, reorder=False): #from sklearn, http://scikit-learn.org, licensed under BSD License
     """Compute Area Under the Curve (AUC) using the trapezoidal rule
 
@@ -108,6 +107,19 @@ def auc(x, y, reorder=False): #from sklearn, http://scikit-learn.org, licensed u
     return area
 
 
+def mae(absolute_error_values):
+    if np is None:
+        return sum(absolute_error_values) / len(absolute_error_values)
+    else:
+        return np.mean(absolute_error_values)
+
+def rmse(squared_error_values):
+    if np is None:
+        return math.sqrt(sum(squared_error_values)/len(squared_error_values))
+    else:
+        return math.sqrt(np.mean(squared_error_values))
+
+
 class ProcessFailed(Exception):
     pass
 
@@ -390,16 +402,25 @@ def __init__(self,  goals = [], observations = [], missing = {}, encoding ='utf-
     def compute(self): 
         assert not False in [type(cls) == int for cls in self.classes]
         ClassEvaluation.compute(self)
-        self.absolute_error = [abs(goal-observation) for goal, observation in self]
-        self.squared_error = [ae**2 for ae in self.absolute_error]
+        self.error = defaultdict(list)
+        self.squared_error = defaultdict(list)
+        for goal, observation in self:
+            self.error[observation].append(abs(goal-observation))
+            self.squared_error[observation].append(abs(goal-observation)**2)
 
-    def mae(self):
+    def mae(self, cls=None):
         if not self.computed: self.compute()
-        return np.mean(self.absolute_error)
-
-    def rmse(self):
+        if cls:
+            return mae(self.error[cls])
+        else:
+            return mae(sum([self.error[x] for x in set(self.goals)], []))
+             
+    def rmse(self, cls=None):
         if not self.computed: self.compute()
-        return math.sqrt(np.mean(self.squared_error))
+        if cls:
+            return rmse(self.squared_error[cls])
+        else:
+            return rmse(sum([self.squared_error[x] for x in set(self.goals)], []))
     
 
 class AbstractExperiment(object):
diff --git a/tests/evaluation.py b/tests/evaluation.py
index aa43aafd..dfe9ae2a 100755
--- a/tests/evaluation.py
+++ b/tests/evaluation.py
@@ -22,7 +22,7 @@
 import unittest
 import random
 
-from pynlpl.evaluation import AbstractExperiment, WPSParamSearch, ExperimentPool, ClassEvaluation
+from pynlpl.evaluation import AbstractExperiment, WPSParamSearch, ExperimentPool, ClassEvaluation, OrdinalEvaluation
 
 class ParamExperiment(AbstractExperiment):
     def defaultparameters(self):
@@ -81,7 +81,18 @@ def test001(self):
         print()
         print(e)
         print(e.confusionmatrix())
-    
+
+class OrdinalEvaluationTest(unittest.TestCase):
+    def setUp(self):
+        self.goals = [1,2,3,4,3,2]
+        self.observations = [4,1,3,4,2,2]
+
+    def test001(self):
+        oe = OrdinalEvaluation(self.goals,self.observations)
+        print(oe.mae())
+        print(oe.mae(2))
+        print(oe.rmse())
+        print(oe.rmse(4))
     
 class ClassEvaluationTest(unittest.TestCase):
     def setUp(self):