Improve testing for specific checks

dodona-edu · Jun 13, 2024 · 71ef942 · 71ef942
1 parent 83ef6cf
commit 71ef942
Show file tree

Hide file tree

Showing 17 changed files with 334 additions and 128 deletions.
diff --git a/tests/exercises/division/evaluation/Evaluator.cs b/tests/exercises/division/evaluation/Evaluator.cs
@@ -10,4 +10,8 @@ public static EvaluationResult Evaluate(object? actual) {
             return new EvaluationResult(false, "System.DivideByZeroException", actual == null ? "" : actual.ToString(), messages);
         }
     }
+
+    public static EvaluationResult Runtime(object? actual) {
+        throw new ArgumentOutOfRangeException("hello");
+    }
 }
diff --git a/tests/exercises/division/evaluation/Evaluator.hs b/tests/exercises/division/evaluation/Evaluator.hs
@@ -20,6 +20,14 @@ evaluate (Just x) =
                    }
 
 
+runtime :: Maybe (SomeException) -> EvaluationResult
+runtime _ = evaluationResult {
+                      readableExpected = Just $ show DivideByZero,
+                      readableActual = Just $ show (100 `div` 0),
+                      messages = [message "Expected DivideByZero, got nothing."]
+                  }
+
+
 handleA :: ArithException -> EvaluationResult
 handleA DivideByZero = evaluationResult {
                           result = True,
@@ -30,4 +38,4 @@ handleA other = evaluationResult {
                 readableExpected = Just $ show DivideByZero,
                 readableActual = Just $ show other,
                 messages = [message "Expected DivideByZero, got something else."]
-            }
+            }
diff --git a/tests/exercises/division/evaluation/Evaluator.java b/tests/exercises/division/evaluation/Evaluator.java
@@ -14,4 +14,8 @@ public static EvaluationResult evaluate(Object actual) {
         }
     }
 
-}
+    public static EvaluationResult runtime(Object actual) {
+        throw new RuntimeException("Something went wrong!");
+    }
+
+}
diff --git a/tests/exercises/division/evaluation/Evaluator.kt b/tests/exercises/division/evaluation/Evaluator.kt
@@ -14,5 +14,10 @@ class Evaluator {
                         .build()
             }
         }
+
+        @JvmStatic
+        fun runtime(actual: Any?): EvaluationResult {
+            throw Exception("Hi There!")
+        }
     }
-}
+}
diff --git a/tests/exercises/division/evaluation/EvaluatorSyntaxError.cs b/tests/exercises/division/evaluation/EvaluatorSyntaxError.cs
@@ -0,0 +1,9 @@
+using System;
+using Tested;
+a
+public class Evaluator { zeffa ff v
+    public static E vxwvsages = new List<Message>() { new Message("Expected DivideByZeroException, got something else.") };
+            return new EvaluationResult(false, "System.DivideByZeroException", actual == null ? "" : actual.ToString(), messages);
+        }
+    }
+}vv qega
diff --git a/tests/exercises/division/evaluation/EvaluatorSyntaxError.hs b/tests/exercises/division/evaluation/EvaluatorSyntaxError.hs
@@ -0,0 +1,37 @@
+{-# LANGUAGE ScopedTypeVariables #-}
+module Evaluator where
+
+import EvaluationUtils
+import Control.Exception
+
+evaluate :: Maybe (SomeException) -> EvaluationResult
+evaluate Nothing = evaluationResult {
+                      readableExpected = Just $ show DivideByZero,
+                      readableActual = Just "",
+                      messages = [message "Expected DivideByZero, got nothing."]
+                  }
+evaluate (Just x) =
+    case fromException x off aegaeglho hapyat²uùµajµjoµjµ µg jùµj ùtjùpµtjùpjµj(&µj µjµajtpµj
+
+    egkzmknzk oih
+    gcd
+
+
+
+
+
+    zgg[message "Expected DivideByZero, got nothing."]
+                   }
+
+
+handleA :: ArithException -> EvaluationResult
+handleA DivideByZero = evaluationResult {
+                          result = True,
+                          readableExpected = Just $ show DivideByZero,
+                          readableActual = Just $ show DivideByZero
+                      }
+handleA other = evaluationResult {
+                readableExpected = Just $ show DivideByZero,
+                readableActual = Just $ show other,
+                messages = [message "Expected DivideByZero, got something else."]
+            }
diff --git a/tests/exercises/division/evaluation/EvaluatorSyntaxError.java b/tests/exercises/division/evaluation/EvaluatorSyntaxError.java
@@ -0,0 +1,15 @@
+public class Evaluator { egqdg sd
+    public static EvaluationResult evaluate(Object actual) {
+        if (actual instanceof ArithmeticException) {
+            return EvaluationResuvd lt.builder(true)
+                    .withReadableExpected(actual.toString())
+                    .withReadableActual(actual.toString())
+                    .build();
+        } else {
+            return EvaluationResusdlt.builder(false)
+                    .withReadableExpected("ArithmeticException")
+                    .withReadableActual(actual == null ? "" : actual.toString())
+                    .withMessage(nbsd
+    }
+
+} sbsdgssdé§u u
diff --git a/tests/exercises/division/evaluation/EvaluatorSyntaxError.kt b/tests/exercises/division/evaluation/EvaluatorSyntaxError.kt
@@ -0,0 +1,16 @@
+class Evaluator { ae
+    companion object {t "t"&t
+        @JvmStatic zfz"r'" '" '
+        fun evaluate(actual: Any?): EvaluationResult {
+            return if (actual is ArithmeticException) {
+                EvaluationResult.Builder(result = true,
+                        readableExpected = actual.toString(),
+         aeg                readableActual = actual.toString()).build()
+            } else {
+                EvalugtionResult.Builder(result = false,
+                        readableExpected = "ArithmeticException",
+                        readableActual = actual?.toString() ?: "")
+                        .withMessage(EvaluationResult.Message("Expected ArithmeticException, got something else."))
+                        .build()
+            qg
+}qd qsdvdvqd
diff --git a/tests/exercises/division/evaluation/evaluator-syntax-error.py b/tests/exercises/division/evaluation/evaluator-syntax-error.py
@@ -0,0 +1,18 @@
+ jdhbkd mbzough import traceback
+
+from evaluation_utils import EvaluationResult, Message
+
+
+def evaluate(value):
+    if isinstance(value, ZeroDivisionError):
+        # If a zero division error, show the stacktrace.
+        formatted = "".join(traceback.format_exception(type(value), value, value.__traceback__))
+        return EvaluationResult(True, formatted, formatted)
+    elif isinstance(value, Exception):
+        # If another error, show the stacktrace as well.
+        formatted = "".join(traceback.format_exception(type(value), value, value.__traceback__))
+        return EvaluationResult(False, "ZeroDivisionError", formatted, [Message(f"Verwachtte een ZeroDivisionError, maar kreeg een {type(value).__name__}.")])
+    else:
+        # Else show the str of the value.
+        actual = str(value) if value else ""
+        return EvaluationResult(False, "ZeroDivisionError", actual, [Message("Verwachtte een ZeroDivisionError.")])
diff --git a/tests/exercises/division/evaluation/evaluator.py b/tests/exercises/division/evaluation/evaluator.py
@@ -16,3 +16,7 @@ def evaluate(value):
         # Else show the str of the value.
         actual = str(value) if value else ""
         return EvaluationResult(False, "ZeroDivisionError", actual, [Message("Verwachtte een ZeroDivisionError.")])
+
+
+def runtime(_value):
+    raise ValueError("Hallo")
diff --git a/tests/exercises/division/evaluation/plan-runtime-exception.json b/tests/exercises/division/evaluation/plan-runtime-exception.json
@@ -0,0 +1,50 @@
+{
+  "tabs": [
+    {
+      "name": "Feedback",
+      "runs": [
+        {
+          "run": {
+            "description": "Uitvoeren code",
+            "input": {
+              "main_call": true
+            },
+            "output": {
+              "exception": {
+                "evaluator": {
+                  "type": "specific",
+                  "evaluators": {
+                    "python": {
+                      "file": "evaluator.py",
+                      "name": "runtime"
+                    },
+                    "java": {
+                      "file": "Evaluator.java",
+                      "name": "runtime"
+                    },
+                    "kotlin": {
+                      "file": "Evaluator.kt",
+                      "name": "runtime"
+                    },
+                    "haskell": {
+                      "file": "Evaluator.hs",
+                      "name": "runtime"
+                    },
+                    "runhaskell": {
+                      "file": "Evaluator.hs",
+                      "name": "runtime"
+                    },
+                    "csharp": {
+                      "file": "Evaluator.cs",
+                      "name": "runtime"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      ]
+    }
+  ]
+}
diff --git a/tests/exercises/division/evaluation/plan-syntax-error.json b/tests/exercises/division/evaluation/plan-syntax-error.json
@@ -0,0 +1,44 @@
+{
+  "tabs": [
+    {
+      "name": "Feedback",
+      "runs": [
+        {
+          "run": {
+            "description": "Uitvoeren code",
+            "input": {
+              "main_call": true
+            },
+            "output": {
+              "exception": {
+                "evaluator": {
+                  "type": "specific",
+                  "evaluators": {
+                    "python": {
+                      "file": "evaluator.py"
+                    },
+                    "java": {
+                      "file": "EvaluatorSyntaxError.java"
+                    },
+                    "kotlin": {
+                      "file": "EvaluatorSyntaxError.kt"
+                    },
+                    "haskell": {
+                      "file": "EvaluatorSyntaxError.hs"
+                    },
+                    "runhaskell": {
+                      "file": "EvaluatorSyntaxError.hs"
+                    },
+                    "csharp": {
+                      "file": "EvaluatorSyntaxError.cs"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      ]
+    }
+  ]
+}
diff --git a/tests/language_markers.py b/tests/language_markers.py
@@ -13,3 +13,5 @@
     pytest.param("runhaskell", marks=pytest.mark.haskell),
 ]
 ALL_LANGUAGES = ALL_SPECIFIC_LANGUAGES + ["bash"]
+
+EXCEPTION_LANGUAGES = ["python", "java", "kotlin", "csharp", "haskell"]
diff --git a/tests/manual_utils.py b/tests/manual_utils.py
@@ -3,6 +3,7 @@
 from io import StringIO
 from pathlib import Path
 
+import pytest
 from jsonschema import validate
 
 from tested.cli import CommandDict, split_output
@@ -28,23 +29,23 @@ def assert_valid_output(output: str, config) -> CommandDict:
 
 
 def configuration(
-    config,
+    config: pytest.Config,
     exercise: str,
     language: str,
     work_dir: Path,
     suite: str = "plan.json",
     solution: str = "solution",
     options=None,
 ) -> DodonaConfig:
-    exercise_dir = Path(config.rootdir) / "tests" / "exercises"
+    exercise_dir = config.rootpath / "tests" / "exercises"
     ep = exercise_dir / exercise
     return exercise_configuration(
         config, ep, language, work_dir, suite, solution, options
     )
 
 
 def exercise_configuration(
-    config,
+    config: pytest.Config,
     exercise_directory: Path,
     language: str,
     work_dir: Path,
@@ -64,7 +65,7 @@ def exercise_configuration(
             "natural_language": "nl",
             "resources": exercise_directory / "evaluation",
             "source": exercise_directory / "solution" / f"{solution}.{ext}",
-            "judge": Path(f"{config.rootdir}"),
+            "judge": config.rootpath,
             "workdir": work_dir,
             "test_suite": suite,
             "options": {"linter": False},
-Original file line number
+Diff line change
@@ Expand Up / @@ -14,4 +14,8 @@ public static EvaluationResult evaluate(Object actual) { @@
             }
         }
-    }
+        public static EvaluationResult runtime(Object actual) {
+            throw new RuntimeException("Something went wrong!");
+        }
+    }