Merge branch 'fkunneman-master'

evaluation.py

@@ -43,7 +43,6 @@
 import datetime
 import os.path
 
-
 def auc(x, y, reorder=False): #from sklearn, http://scikit-learn.org, licensed under BSD License
     """Compute Area Under the Curve (AUC) using the trapezoidal rule
 
@@ -108,6 +107,19 @@ def auc(x, y, reorder=False): #from sklearn, http://scikit-learn.org, licensed u
     return area
 
 
+def mae(absolute_error_values):
+    if np is None:
+        return sum(absolute_error_values) / len(absolute_error_values)
+    else:
+        return np.mean(absolute_error_values)
+
+def rmse(squared_error_values):
+    if np is None:
+        return math.sqrt(sum(squared_error_values)/len(squared_error_values))
+    else:
+        return math.sqrt(np.mean(squared_error_values))
+
+
 class ProcessFailed(Exception):
     pass
 
@@ -390,23 +402,25 @@ def __init__(self,  goals = [], observations = [], missing = {}, encoding ='utf-
     def compute(self):
         assert not False in [type(cls) == int for cls in self.classes]
         ClassEvaluation.compute(self)
-        self.absolute_error = [abs(goal-observation) for goal, observation in self]
-        self.squared_error = [ae**2 for ae in self.absolute_error]
+        self.error = defaultdict(list)
+        self.squared_error = defaultdict(list)
+        for goal, observation in self:
+            self.error[observation].append(abs(goal-observation))
+            self.squared_error[observation].append(abs(goal-observation)**2)
 
-    def mae(self):
+    def mae(self, cls=None):
         if not self.computed: self.compute()
-        if np is None:
-            return sum(self.absolute_error) / len(self.absolute_error)
+        if cls:
+            return mae(self.error[cls])
         else:
-            return np.mean(self.absolute_error)
-
-    def rmse(self):
+            return mae(sum([self.error[x] for x in set(self.goals)], []))
+             
+    def rmse(self, cls=None):
         if not self.computed: self.compute()
-        if np is None:
-            return math.sqrt(sum(self.squared_error)/len(self.squared_error))
+        if cls:
+            return rmse(self.squared_error[cls])
         else:
-            return math.sqrt(np.mean(self.squared_error))
-
+            return rmse(sum([self.squared_error[x] for x in set(self.goals)], []))
 
 class AbstractExperiment(object):
 

tests/evaluation.py

@@ -22,7 +22,7 @@
 import unittest
 import random
 
-from pynlpl.evaluation import AbstractExperiment, WPSParamSearch, ExperimentPool, ClassEvaluation
+from pynlpl.evaluation import AbstractExperiment, WPSParamSearch, ExperimentPool, ClassEvaluation, OrdinalEvaluation
 
 class ParamExperiment(AbstractExperiment):
     def defaultparameters(self):
@@ -81,7 +81,18 @@ def test001(self):
         print()
         print(e)
         print(e.confusionmatrix())
-
+
+class OrdinalEvaluationTest(unittest.TestCase):
+    def setUp(self):
+        self.goals = [1,2,3,4,3,2]
+        self.observations = [4,1,3,4,2,2]
+
+    def test001(self):
+        oe = OrdinalEvaluation(self.goals,self.observations)
+        print(oe.mae())
+        print(oe.mae(2))
+        print(oe.rmse())
+        print(oe.rmse(4))
 
 class ClassEvaluationTest(unittest.TestCase):
     def setUp(self):