Merge pull request #439 from dodona-edu/feature/dsl-in-oracles

Support DSL values in oracles

tested/dsl/translate_parser.py

@@ -303,14 +303,17 @@ def _convert_file(link_file: YamlDict) -> FileUrl:
 
 
 def _convert_custom_check_oracle(stream: dict) -> CustomCheckOracle:
+    converted_args = []
+    for v in stream.get("arguments", []):
+        cv = _convert_yaml_value(v)
+        assert isinstance(cv, Value)
+        converted_args.append(cv)
     return CustomCheckOracle(
         language=stream["language"],
         function=EvaluationFunction(
             file=stream["file"], name=stream.get("name", "evaluate")
         ),
-        arguments=[
-            parse_string(v, is_return=True) for v in stream.get("arguments", [])
-        ],
+        arguments=converted_args,
     )
 
 
@@ -338,27 +341,30 @@ def _convert_text_output_channel(
         raise TypeError(f"Unknown text oracle type: {stream['oracle']}")
 
 
-def _convert_advanced_value_output_channel(stream: YamlObject) -> ValueOutputChannel:
+def _convert_yaml_value(stream: YamlObject) -> Value | None:
     if isinstance(stream, YamlValue):
         # A normal yaml type tagged explicitly.
         value = _convert_value(stream.value)
-        assert isinstance(value, Value)
-        return ValueOutputChannel(value=value)
-    if isinstance(stream, (int, float, bool, TestedType, list, set)):
+    elif isinstance(stream, (int, float, bool, TestedType, list, set)):
         # Simple values where no confusion is possible.
         value = _convert_value(stream)
-        assert isinstance(value, Value)
-        return ValueOutputChannel(value=value)
     elif isinstance(stream, str):
         # A normal YAML string is considered a "Python" string.
         value = parse_string(stream, is_return=True)
-        assert isinstance(value, Value)
-        return ValueOutputChannel(value=value)
+    else:
+        return None
+    assert isinstance(value, Value)
+    return value
+
+
+def _convert_advanced_value_output_channel(stream: YamlObject) -> ValueOutputChannel:
+    yaml_value = _convert_yaml_value(stream)
+    if yaml_value:
+        return ValueOutputChannel(value=yaml_value)
     else:
         # We have an object, which means we have an output channel.
         assert isinstance(stream, dict)
-        assert isinstance(stream["value"], str)
-        value = parse_string(stream["value"], is_return=True)
+        value = _convert_yaml_value(stream["value"])
         assert isinstance(value, Value)
         if "oracle" not in stream or stream["oracle"] == "builtin":
             return ValueOutputChannel(value=value)

tested/internationalization/nl.yaml

@@ -77,7 +77,7 @@ nl:
         Contacteer de lesgever.
       unsupported: "Evaluation in %{lang} wordt niet ondersteund."
       unknown:
-        compilation: "Ongekende compilatie fout"
+        compilation: "Ongekende compilatiefout"
       produced:
         stderr: "De evaluatiecode produceerde volgende uitvoer op stderr:"
         stdout: "De evaluatiecode produceerde volgende uitvoer op stdout:"
@@ -86,7 +86,7 @@ nl:
   languages:
     config:
       unknown:
-        compilation: "Ongekende compilatie fout"
+        compilation: "Ongekende compilatiefout"
     generator:
       missing:
         input: "Geen invoer gevonden."

tested/judge/evaluation.py

@@ -33,8 +33,10 @@
     get_readable_input,
 )
 from tested.oracles import get_oracle
+from tested.oracles.common import OracleResult
 from tested.testsuite import (
     Context,
+    EmptyChannel,
     ExceptionOutput,
     ExceptionOutputChannel,
     ExitCodeOutputChannel,
@@ -113,7 +115,7 @@ def _evaluate_channel(
 
     # Decide if we should show this channel or not.
     is_correct = status.enum == Status.CORRECT
-    should_report_case = should_show(output, channel)
+    should_report_case = should_show(output, channel, evaluation_result)
 
     if not should_report_case and is_correct:
         # We do report that a test is correct, to set the status.
@@ -363,14 +365,17 @@ def _link_files_message(link_files: Collection[FileUrl]) -> AppendMessage:
     return AppendMessage(message=message)
 
 
-def should_show(test: OutputChannel, channel: Channel) -> bool:
+def should_show(
+    test: OutputChannel, channel: Channel, result: OracleResult | None = None
+) -> bool:
     """
     Determine if the channel should be shown, without accounting for the actual
     value. This function answers the question: "Assuming the actual value is
     correct, should we show this output channel?".
 
     :param test: The output for the channel from the test suite.
     :param channel: The channel.
+    :param result: The result of the evaluation.
 
     :return: True if the channel should be shown, false otherwise.
     """
@@ -390,7 +395,15 @@ def should_show(test: OutputChannel, channel: Channel) -> bool:
     elif channel == Channel.RETURN:
         assert isinstance(test, ValueOutput)
         # We don't show the channel if we ignore it.
-        return not isinstance(test, IgnoredChannel)
+        if isinstance(test, IgnoredChannel):
+            return False
+        if (
+            isinstance(test, EmptyChannel)
+            and result
+            and result.result.enum == Status.CORRECT
+        ):
+            return False
+        return True
     elif channel == Channel.EXCEPTION:
         assert isinstance(test, ExceptionOutput)
         return not isinstance(test, SpecialOutputChannel)

tested/judge/programmed.py

@@ -17,12 +17,9 @@
 from tested.internationalization import get_i18n_string
 from tested.judge.execution import execute_file, filter_files
 from tested.judge.utils import BaseExecutionResult, copy_from_paths_to_path, run_command
-from tested.languages.generation import (
-    custom_oracle_arguments,
-    generate_custom_evaluator,
-    generate_statement,
-)
-from tested.serialisation import BooleanEvalResult, EvalResult, Value
+from tested.languages.generation import generate_custom_evaluator, generate_statement
+from tested.oracles.common import BooleanEvalResult
+from tested.serialisation import FunctionCall, FunctionType, Value
 from tested.testsuite import CustomCheckOracle
 from tested.utils import get_identifier
 
@@ -34,7 +31,7 @@ def evaluate_programmed(
     evaluator: CustomCheckOracle,
     expected: Value,
     actual: Value,
-) -> BaseExecutionResult | EvalResult:
+) -> BaseExecutionResult | BooleanEvalResult:
     """
     Run the custom evaluation. Concerning structure and execution, the custom
     oracle is very similar to the execution of the whole evaluation. It a
@@ -181,7 +178,7 @@ def _evaluate_python(
     oracle: CustomCheckOracle,
     expected: Value,
     actual: Value,
-) -> EvalResult:
+) -> BooleanEvalResult:
     """
     Run an evaluation in Python. While the templates are still used to generate
     code, they are not executed in a separate process, but inside python itself.
@@ -213,17 +210,15 @@ def _evaluate_python(
     exec(evaluator_code, global_env)
 
     # Call the oracle.
-    literal_expected = generate_statement(eval_bundle, expected)
-    literal_actual = generate_statement(eval_bundle, actual)
-    arguments = custom_oracle_arguments(oracle)
-    literal_arguments = generate_statement(eval_bundle, arguments)
+    check_function_call = FunctionCall(
+        type=FunctionType.FUNCTION,
+        name=oracle.function.name,
+        arguments=[expected, actual, *oracle.arguments],
+    )
+    literal_function_call = generate_statement(eval_bundle, check_function_call)
 
     with _catch_output() as (stdout_, stderr_):
-        exec(
-            f"__tested_test__result = {oracle.function.name}("
-            f"{literal_expected}, {literal_actual}, {literal_arguments})",
-            global_env,
-        )
+        exec(f"__tested_test__result = {literal_function_call}", global_env)
 
     stdout_ = stdout_.getvalue()
     stderr_ = stderr_.getvalue()
@@ -267,13 +262,12 @@ def _evaluate_python(
                 permission=Permission.STAFF,
             )
         )
-        return EvalResult(
+        return BooleanEvalResult(
             result=Status.INTERNAL_ERROR,
             readable_expected=None,
             readable_actual=None,
             messages=messages,
         )
 
-    assert isinstance(result_, BooleanEvalResult)
     result_.messages.extend(messages)
-    return result_.as_eval_result()
+    return result_

tested/languages/bash/generators.py

@@ -238,7 +238,7 @@ def convert_encoder(values: list[Value]) -> str:
 }}
 
 function send_value {{
-    echo "{{\\"type\\": \\"text\\", \\"data\\": $(json_escape "$1")}}"
+    echo "{{\\"type\\": \\"text\\", \\"data\\": $(json_escape "$1")}}␞"
 }}
 """
     for value in values:

tested/languages/c/generators.py

@@ -322,6 +322,6 @@ def convert_encoder(values: list[Value]) -> str:
 """
     for value in values:
         result += " " * 4 + f"write_value(stdout, {convert_value(value)});\n"
-        result += " " * 4 + 'printf("\\n");\n'
+        result += " " * 4 + 'printf("␞");\n'
     result += "}\n"
     return result

tested/languages/c/templates/evaluation_result.h

@@ -15,6 +15,8 @@ typedef struct EvaluationResult {
     char* readableActual;
     size_t nrOfMessages;
     Message** messages;
+    char* dslExpected;
+    char* dslActual;
 } EvaluationResult;
 
 

tested/languages/csharp/generators.py

@@ -478,7 +478,7 @@ def convert_encoder(values: list[Value]) -> str:
 
     for value in values:
         result += " " * 6 + f"Values.WriteValue(stdout, {convert_value(value)});"
-        result += " " * 6 + 'stdout.Write("\\n");\n'
+        result += " " * 6 + 'stdout.Write("␞");\n'
 
     result += "}\n}\n}\n"
     return result

tested/languages/csharp/templates/EvaluationResult.cs

@@ -3,7 +3,7 @@
 
 namespace Tested {
   public record Message(string Description, string Format = "text", string? Permission = null);
-  public record EvaluationResult(bool Result, string? ReadableExpected = null, string? ReadableActual = null, List<Message>? Messages = null)
+  public record EvaluationResult(bool Result, string? ReadableExpected = null, string? ReadableActual = null, List<Message>? Messages = null, string? DslExpected = null, string? DslActual = null)
   {
     public List<Message> Messages { get; init; } = Messages ?? new List<Message>();
   }

tested/languages/generation.py

@@ -17,7 +17,7 @@
 from pygments.lexers import get_lexer_by_name
 
 from tested.configs import Bundle
-from tested.datatypes import AllTypes, BasicObjectTypes, BasicSequenceTypes
+from tested.datatypes import AllTypes, BasicObjectTypes
 from tested.dodona import ExtendedMessage
 from tested.internationalization import get_i18n_string
 from tested.judge.planning import PlannedExecutionUnit
@@ -40,7 +40,6 @@
     Expression,
     FunctionType,
     Identifier,
-    SequenceType,
     Statement,
     Value,
     VariableType,
@@ -294,12 +293,6 @@ def generate_selector(
     return selector_filename
 
 
-def custom_oracle_arguments(oracle: CustomCheckOracle) -> Value:
-    return SequenceType(
-        type=BasicSequenceTypes.SEQUENCE, data=oracle.arguments  # pyright: ignore
-    )
-
-
 def generate_custom_evaluator(
     bundle: Bundle,
     destination: Path,
@@ -321,13 +314,12 @@ def generate_custom_evaluator(
     evaluator_name = conventionalize_namespace(
         bundle.language, evaluator.function.file.stem
     )
-    arguments = custom_oracle_arguments(evaluator)
 
     function = PreparedFunctionCall(
         type=FunctionType.FUNCTION,
         namespace=Identifier(evaluator_name),
         name=evaluator.function.name,
-        arguments=[expected_value, actual_value, arguments],
+        arguments=[expected_value, actual_value, *evaluator.arguments],
         has_root_namespace=False,
     )
 

tested/languages/haskell/generators.py

@@ -350,5 +350,5 @@ def convert_encoder(values: list[Value]) -> str:
 
     for value in values:
         result += indent + f"sendValueH stdout ({convert_value(value)})\n"
-        result += indent + 'putStr "\\n"\n'
+        result += indent + 'putStr "␞"\n'
     return result

tested/languages/haskell/templates/EvaluationUtils.hs

@@ -11,7 +11,9 @@ data EvaluationResult = EvaluationResult {
     result :: Bool,
     readableExpected :: Maybe (String),
     readableActual :: Maybe (String),
-    messages :: [Message]
+    messages :: [Message],
+    dslExpected :: Maybe (String),
+    dslActual :: Maybe (String)
 } deriving Show
 
 message description = Message {
@@ -24,5 +26,7 @@ evaluationResult = EvaluationResult {
     result = False,
     readableExpected = Nothing,
     readableActual = Nothing,
+    dslExpected = Nothing,
+    dslActual = Nothing,
     messages = []
 }

tested/languages/java/generators.py

@@ -450,7 +450,7 @@ def convert_encoder(values: list[Value]) -> str:
 
     for value in values:
         result += " " * 8 + f"Values.send(writer, {convert_value(value)});\n"
-        result += " " * 8 + 'writer.write("\\n");\n'
+        result += " " * 8 + 'writer.write("␞");\n'
 
     result += " " * 8 + "writer.close();\n"
     result += " " * 4 + "}\n"

tested/languages/java/templates/EvaluationResult.java

@@ -6,12 +6,16 @@ public class EvaluationResult {
     public final boolean result;
     public final String readableExpected;
     public final String readableActual;
+    public final String dslExpected;
+    public final String dslActual;
     public final List<Message> messages;
 
-    private EvaluationResult(boolean result, String readableExpected, String readableActual, List<Message> messages) {
+    private EvaluationResult(boolean result, String readableExpected, String readableActual, String dslExpected, String dslActual, List<Message> messages) {
         this.result = result;
         this.readableExpected = readableExpected;
         this.readableActual = readableActual;
+        this.dslExpected = dslExpected;
+        this.dslActual = dslActual;
         this.messages = messages;
     }
 
@@ -44,6 +48,8 @@ public static class Builder {
         private final boolean result;
         private String readableExpected = null;
         private String readableActual = null;
+        private String dslExpected = null;
+        private String dslActual = null;
         private final List<Message> messages = new ArrayList<>();
 
         private Builder(boolean result) {
@@ -59,6 +65,16 @@ public Builder withReadableActual(String readableActual) {
             this.readableActual = readableActual;
             return this;
         }
+
+        public Builder withDslExpected(String dslExpected) {
+            this.dslExpected = dslExpected;
+            return this;
+        }
+
+        public Builder withDslActual(String dslActual) {
+            this.dslActual = dslActual;
+            return this;
+        }
 
         public Builder withMessages(List<Message> messages) {
             this.messages.addAll(messages);
@@ -71,7 +87,7 @@ public Builder withMessage(Message message) {
         }
 
         public EvaluationResult build() {
-            return new EvaluationResult(result, readableExpected, readableActual, messages);
+            return new EvaluationResult(result, readableExpected, readableActual, dslExpected, dslActual, messages);
         }
     }
 }