Rename evaluator to oracle

tested/configs.py

@@ -56,7 +56,7 @@ class Options:
     """
     optimized: bool = True
     """
-    If the custom Python evaluator should be optimized or not.
+    If the Python oracles should be optimized or not.
     """
     compiler_optimizations: bool = False
     """

tested/dsl/schema.json

@@ -248,7 +248,7 @@
       "oneOf" : [
         {
           "properties" : {
-            "evaluator" : {
+            "oracle" : {
               "type" : "string",
               "enum" : [
                 "builtin"
@@ -258,15 +258,15 @@
         },
         {
           "required" : [
-            "evaluator",
+            "oracle",
             "language",
             "file"
           ],
           "properties" : {
-            "evaluator" : {
+            "oracle" : {
               "type" : "string",
               "enum" : [
-                "custom"
+                "custom_check"
               ]
             },
             "language" : {
@@ -314,7 +314,7 @@
           "oneOf" : [
             {
               "properties" : {
-                "evaluator" : {
+                "oracle" : {
                   "type" : "string",
                   "enum" : [
                     "builtin"
@@ -324,15 +324,15 @@
             },
             {
               "required" : [
-                "evaluator",
+                "oracle",
                 "language",
                 "file"
               ],
               "properties" : {
-                "evaluator" : {
+                "oracle" : {
                   "type" : "string",
                   "enum" : [
-                    "custom"
+                    "custom_check"
                   ]
                 },
                 "language" : {

tested/dsl/translate_parser.py

@@ -27,16 +27,16 @@
 )
 from tested.testsuite import (
     Context,
+    CustomCheckOracle,
     EmptyChannel,
     EvaluationFunction,
     ExceptionOutputChannel,
     ExitCodeOutputChannel,
     ExpectedException,
     FileUrl,
-    GenericTextEvaluator,
+    GenericTextOracle,
     MainInput,
     Output,
-    ProgrammedEvaluator,
     Suite,
     Tab,
     Testcase,
@@ -147,8 +147,8 @@ def _convert_file(link_file: YamlDict) -> FileUrl:
     return FileUrl(name=link_file["name"], url=link_file["url"])
 
 
-def _convert_programmed_evaluator(stream: dict) -> ProgrammedEvaluator:
-    return ProgrammedEvaluator(
+def _convert_custom_check_oracle(stream: dict) -> CustomCheckOracle:
+    return CustomCheckOracle(
         language=stream["language"],
         function=EvaluationFunction(
             file=stream["file"], name=stream.get("name", "evaluate")
@@ -165,23 +165,21 @@ def _convert_text_output_channel(
     if isinstance(stream, str):
         data = stream
         config = config.get(config_name, {})
-        return TextOutputChannel(
-            data=data, evaluator=GenericTextEvaluator(options=config)
-        )
+        return TextOutputChannel(data=data, oracle=GenericTextOracle(options=config))
     else:
         assert isinstance(stream, dict)
         data = str(stream["data"])
-        if "evaluator" not in stream or stream["evaluator"] == "builtin":
+        if "oracle" not in stream or stream["oracle"] == "builtin":
             existing_config = config.get(config_name, {})
             config = _deepen_config_level(stream, existing_config)
             return TextOutputChannel(
-                data=data, evaluator=GenericTextEvaluator(options=config)
+                data=data, oracle=GenericTextOracle(options=config)
             )
-        elif stream["evaluator"] == "custom":
+        elif stream["oracle"] == "custom_check":
             return TextOutputChannel(
-                data=data, evaluator=_convert_programmed_evaluator(stream)
+                data=data, oracle=_convert_custom_check_oracle(stream)
             )
-        raise TypeError(f"Unknown text evaluator type: {stream['evaluator']}")
+        raise TypeError(f"Unknown text oracle type: {stream['oracle']}")
 
 
 def _convert_advanced_value_output_channel(stream: YamlObject) -> ValueOutputChannel:
@@ -194,14 +192,14 @@ def _convert_advanced_value_output_channel(stream: YamlObject) -> ValueOutputCha
         assert isinstance(stream["value"], str)
         value = parse_string(stream["value"], is_return=True)
         assert isinstance(value, Value)
-        if "evaluator" not in stream or stream["evaluator"] == "builtin":
+        if "oracle" not in stream or stream["oracle"] == "builtin":
             return ValueOutputChannel(value=value)
-        elif stream["evaluator"] == "custom":
+        elif stream["oracle"] == "custom_check":
             return ValueOutputChannel(
                 value=value,
-                evaluator=_convert_programmed_evaluator(stream),
+                oracle=_convert_custom_check_oracle(stream),
             )
-        raise TypeError(f"Unknown value evaluator type: {stream['evaluator']}")
+        raise TypeError(f"Unknown value oracle type: {stream['oracle']}")
 
 
 def _convert_testcase(testcase: YamlDict, previous_config: dict) -> Testcase:

tested/features.py

@@ -148,16 +148,16 @@ def is_supported(language: "Language") -> bool:
             _logger.warning(f"Test suite requires unsupported type {t}")
             return False
 
-    # Check language-specific evaluators
+    # Check language-specific oracles
     for tab in language.config.suite.tabs:
         assert tab.contexts is not None
         for context in tab.contexts:
             for testcase in context.testcases:
-                languages = testcase.output.get_specific_eval_languages()
+                languages = testcase.output.get_specific_oracle_languages()
                 if languages is not None:
                     if language.config.dodona.programming_language not in languages:
                         _logger.warning(
-                            f"Specific evaluators are available only in "
+                            f"Language-specific oracles are available only in "
                             f"{languages}!"
                         )
                         return False

tested/judge/core.py

@@ -334,10 +334,10 @@ def _generate_files(
                 execution_unit=unit,
                 execution_name=exec_name,
             )
-            # Copy evaluators to the directory.
+            # Copy functions to the directory.
             for evaluator in evaluators:
                 source = Path(bundle.config.resources) / evaluator
-                _logger.debug("Copying evaluator from %s to %s", source, common_dir)
+                _logger.debug("Copying oracle from %s to %s", source, common_dir)
                 shutil.copy2(source, common_dir)
             dependencies.extend(evaluators)
             dependencies.append(generated)

tested/judge/evaluation.py

@@ -25,7 +25,6 @@
     StatusMessage,
     Update,
 )
-from tested.evaluators import get_evaluator
 from tested.internationalization import get_i18n_string
 from tested.judge.collector import OutputManager, TestcaseCollector
 from tested.judge.execution import ContextResult
@@ -34,6 +33,7 @@
     generate_statement,
     get_readable_input,
 )
+from tested.oracles import get_oracle
 from tested.testsuite import (
     Context,
     ExceptionOutput,
@@ -103,10 +103,10 @@ def _evaluate_channel(
 
     :return: True if successful, otherwise False.
     """
-    evaluator = get_evaluator(
+    evaluator = get_oracle(
         bundle, context_directory, output, unexpected_status=unexpected_status
     )
-    # Run the evaluator.
+    # Run the oracle.
     evaluation_result = evaluator(output, actual if actual else "")
     status = evaluation_result.result
 
@@ -260,7 +260,7 @@ def evaluate_context_results(
         inlined_files = inlined_files.union(seen)
         t_col = TestcaseCollector(StartTestcase(description=readable_input))
 
-        # Get the evaluators
+        # Get the functions
         output = testcase.output
 
         # Get the values produced by the execution. If there are no values,
@@ -420,7 +420,7 @@ def should_show(test: OutputChannel, channel: Channel) -> bool:
 def guess_expected_value(bundle: Bundle, test: OutputChannel) -> str:
     """
     Try and get the expected value for an output channel. In some cases, such as
-    a programmed or language specific evaluator, there will be no expected value
+    a programmed or language specific oracle, there will be no expected value
     available in the test suite. In that case, we use an empty string.
 
     :param bundle: Configuration bundle.