Merge pull request #457 from dodona-edu/switch-return-values

Update default interpretation of YAML strings and objects

tested/dsl/schema.json

@@ -378,10 +378,7 @@
             },
             "arguments" : {
               "type" : "array",
-              "description" : "List of 'Python' values to use as arguments to the function.",
-              "items" : {
-                "type" : "string"
-              }
+              "description" : "List of YAML (or tagged expression) values to use as arguments to the function."
             }
           }
         }
@@ -446,10 +443,7 @@
                 },
                 "arguments" : {
                   "type" : "array",
-                  "description" : "List of 'Python' values to use as arguments to the function.",
-                  "items" : {
-                    "type" : "string"
-                  }
+                  "description" : "List of YAML (or tagged expression) values to use as arguments to the function."
                 }
               }
             }

tested/dsl/schema_draft7.json

@@ -373,10 +373,7 @@
             },
             "arguments" : {
               "type" : "array",
-              "description" : "List of 'Python' values to use as arguments to the function.",
-              "items" : {
-                "type" : "string"
-              }
+              "description" : "List of YAML (or tagged expression) values to use as arguments to the function."
             }
           }
         }
@@ -440,10 +437,7 @@
                 },
                 "arguments" : {
                   "type" : "array",
-                  "description" : "List of 'Python' values to use as arguments to the function.",
-                  "items" : {
-                    "type" : "string"
-                  }
+                  "description" : "List of YAML (or tagged expression) values to use as arguments to the function."
                 }
               }
             }

tested/dsl/translate_parser.py

@@ -65,9 +65,7 @@
 )
 from tested.utils import get_args
 
-OptionDict = dict[str, int | bool]
 YamlDict = dict[str, "YamlObject"]
-YamlObject = YamlDict | list | bool | float | int | str | None
 
 
 @define
@@ -77,8 +75,19 @@ class TestedType:
 
 
 @define
-class YamlValue:
-    value: Any
+class ExpressionString:
+    expression: str
+
+
+@define
+class ReturnOracle:
+    value: YamlDict
+
+
+OptionDict = dict[str, int | bool]
+YamlObject = (
+    YamlDict | list | bool | float | int | str | None | ExpressionString | ReturnOracle
+)
 
 
 def _parse_yaml_value(loader: yaml.Loader, node: yaml.Node) -> Any:
@@ -98,8 +107,18 @@ def _custom_type_constructors(loader: yaml.Loader, node: yaml.Node) -> TestedTyp
     return TestedType(type=tested_tag, value=base_result)
 
 
-def _yaml_value_constructor(loader: yaml.Loader, node: yaml.Node) -> YamlValue:
-    return YamlValue(value=_parse_yaml_value(loader, node))
+def _expression_string(loader: yaml.Loader, node: yaml.Node) -> ExpressionString:
+    result = _parse_yaml_value(loader, node)
+    assert isinstance(result, str), f"An expression must be a string, got {result}"
+    return ExpressionString(expression=result)
+
+
+def _return_oracle(loader: yaml.Loader, node: yaml.Node) -> ReturnOracle:
+    result = _parse_yaml_value(loader, node)
+    assert isinstance(
+        result, dict
+    ), f"A custom oracle must be an object, got {result} which is a {type(result)}."
+    return ReturnOracle(value=result)
 
 
 def _parse_yaml(yaml_stream: str) -> YamlObject:
@@ -110,8 +129,8 @@ def _parse_yaml(yaml_stream: str) -> YamlObject:
     for types in get_args(AllTypes):
         for actual_type in types:
             yaml.add_constructor("!" + actual_type, _custom_type_constructors, loader)
-    yaml.add_constructor("!v", _yaml_value_constructor, loader)
-    yaml.add_constructor("!value", _yaml_value_constructor, loader)
+    yaml.add_constructor("!expression", _expression_string, loader)
+    yaml.add_constructor("!oracle", _return_oracle, loader)
 
     try:
         return yaml.load(yaml_stream, loader)
@@ -352,15 +371,12 @@ def _convert_text_output_channel(stream: YamlObject) -> TextOutputChannel:
 
 
 def _convert_yaml_value(stream: YamlObject) -> Value | None:
-    if isinstance(stream, YamlValue):
-        # A normal yaml type tagged explicitly.
-        value = _convert_value(stream.value)
-    elif isinstance(stream, (int, float, bool, TestedType, list, set)):
+    if isinstance(stream, ExpressionString):
+        # We have an expression string.
+        value = parse_string(stream.expression, is_return=True)
+    elif isinstance(stream, (int, float, bool, TestedType, list, set, str, dict)):
         # Simple values where no confusion is possible.
         value = _convert_value(stream)
-    elif isinstance(stream, str):
-        # A normal YAML string is considered a "Python" string.
-        value = parse_string(stream, is_return=True)
     else:
         return None
     assert isinstance(
@@ -370,22 +386,21 @@ def _convert_yaml_value(stream: YamlObject) -> Value | None:
 
 
 def _convert_advanced_value_output_channel(stream: YamlObject) -> ValueOutputChannel:
-    yaml_value = _convert_yaml_value(stream)
-    if yaml_value:
-        return ValueOutputChannel(value=yaml_value)
-    else:
-        # We have an object, which means we have an output channel.
-        assert isinstance(stream, dict)
-        value = _convert_yaml_value(stream["value"])
-        assert isinstance(value, Value)
-        if "oracle" not in stream or stream["oracle"] == "builtin":
+    if isinstance(stream, ReturnOracle):
+        return_object = stream.value
+        value = _convert_yaml_value(return_object["value"])
+        assert isinstance(value, Value), "You must specify a value for a return oracle."
+        if "oracle" not in return_object or return_object["oracle"] == "builtin":
             return ValueOutputChannel(value=value)
-        elif stream["oracle"] == "custom_check":
+        elif return_object["oracle"] == "custom_check":
             return ValueOutputChannel(
                 value=value,
-                oracle=_convert_custom_check_oracle(stream),
+                oracle=_convert_custom_check_oracle(return_object),
             )
-        raise TypeError(f"Unknown value oracle type: {stream['oracle']}")
+        raise TypeError(f"Unknown value oracle type: {return_object['oracle']}")
+    else:
+        yaml_value = _convert_yaml_value(stream)
+        return ValueOutputChannel(value=yaml_value)
 
 
 def _validate_testcase_combinations(testcase: YamlDict):

tests/exercises/echo-function/evaluation/expected_return_and_got_some.yaml

@@ -1,4 +1,4 @@
 - tab: "My tab"
   testcases:
     - expression: 'echo("input")'
-      return: !v "input"
+      return: "input"

tests/exercises/echo-function/evaluation/expected_return_but_got_none.yaml

@@ -1,4 +1,4 @@
 - tab: "My tab"
   testcases:
     - expression: 'no_echo("input")'
-      return: !v "input"
+      return: "input"

tests/exercises/echo-function/evaluation/one-language-literals.yaml

@@ -9,4 +9,4 @@
         kotlin: "toString(1+1)"
         python: "submission.to_string(1+1)"
         csharp: "Submission.toString(1+1)"
-      return: !v "2"
+      return: "2"

tests/exercises/echo-function/evaluation/one-nested.yaml

@@ -1,4 +1,4 @@
 - tab: "My tab"
   testcases:
     - expression: 'echo(echo("input"))'
-      return: !v "input"
+      return: "input"

tests/exercises/global/evaluation/plan.yaml

@@ -1,7 +1,7 @@
 - tab: "Global variable"
   testcases:
     - expression: "GLOBAL_VAR"
-      return: !v "GLOBAL"
+      return: "GLOBAL"
       description:
         description: "Hallo"
         format: "code"

tests/exercises/objects/evaluation/missing_key_types.yaml

@@ -1,4 +1,4 @@
 - tab: "Feedback"
   testcases:
     - expression: '{{"a"}: [int32(1)], {"b"}: "a.txt"}'
-      return: '{{"a"}: [int32(1)], {"b"}: "a.txt"}'
+      return: !expression '{{"a"}: [int32(1)], {"b"}: "a.txt"}'

tests/exercises/objects/evaluation/no-test.yaml

@@ -1,11 +1,11 @@
 - tab: "Feedback"
   testcases:
     - statement: '{["a", "b"], ["c"]}'
-      return: '{["a", "b"], ["a"]}'
+      return: !expression '{["a", "b"], ["a"]}'
     - statement: 'x = {{"a"}: [int16(1)], {"b"}: [int16(0)]}'
     - statement: 'x = {{"a"}: [int32(1)], {"b"}: "a"}'
     - expression: '{{"a"}: [int32(1)], {"b"}: "a.txt"}'
-      return: '{{"a"}: [int32(1)], {"b"}: "a.txt"}'
+      return: !expression '{{"a"}: [int32(1)], {"b"}: "a.txt"}'
       files:
         - name: "a.txt"
           url: "a.txt"

tests/test_dsl_yaml.py

@@ -261,7 +261,7 @@ def test_statements():
         data: "New safe"
         config: *stdout
     - expression: 'safe.content()'
-      return: !v "Ignore whitespace"
+      return: "Ignore whitespace"
   - testcases:
     - statement: 'safe: Safe = Safe(uint8(5))'
       stdout:
@@ -270,7 +270,7 @@ def test_statements():
           <<: *stdout
           ignoreWhitespace: false
     - expression: 'safe.content()'
-      return: 'uint8(5)'
+      return: !expression 'uint8(5)'
     """
     json_str = translate_to_test_suite(yaml_str)
     suite = parse_test_suite(json_str)
@@ -385,7 +385,7 @@ def test_invalid_yaml():
     stderr: []
     testcases:
     - statement: 'data = () ()'
-      return: '() {}'
+      return: !expression '() {}'
     """
     with pytest.raises(Exception):
         translate_to_test_suite(yaml_str)
@@ -408,7 +408,7 @@ def test_statement_with_yaml_dict():
 - tab: "Feedback"
   testcases:
   - expression: "get_dict()"
-    return: !v
+    return:
         alpha: 5
         beta: 6
 """
@@ -462,7 +462,7 @@ def test_expression_raw_return():
   contexts:
     - testcases:
         - expression: 'test()'
-          return: '[(4, 4), (4, 3), (4, 2), (4, 1), (4, 0), (3, 0), (3, 1), (4, 1)]'
+          return: !expression '[(4, 4), (4, 3), (4, 2), (4, 1), (4, 0), (3, 0), (3, 1), (4, 1)]'
 """
     json_str = translate_to_test_suite(yaml_str)
     suite = parse_test_suite(json_str)
@@ -494,7 +494,7 @@ def test_empty_constructor(function_name, result):
   contexts:
     - testcases:
         - expression: 'test()'
-          return: '{function_name}()'
+          return: !expression '{function_name}()'
 """
     json_str = translate_to_test_suite(yaml_str)
     suite = parse_test_suite(json_str)
@@ -524,7 +524,7 @@ def test_empty_constructor_with_param(function_name, result):
   contexts:
     - testcases:
         - expression: 'test()'
-          return: '{function_name}([])'
+          return: !expression '{function_name}([])'
 """
     json_str = translate_to_test_suite(yaml_str)
     suite = parse_test_suite(json_str)
@@ -598,7 +598,7 @@ def test_text_custom_checks_correct():
                 language: "python"
                 file: "test.py"
                 name: "evaluate_test"
-                arguments: ["'yes'", "5", "set([5, 5])"]
+                arguments: [!expression "'yes'", 5, !expression "set([5, 5])"]
     """
     json_str = translate_to_test_suite(yaml_str)
     suite = parse_test_suite(json_str)
@@ -635,8 +635,8 @@ def test_value_built_in_checks_implied():
       contexts:
         - testcases:
             - expression: 'test()'
-              return:
-                value: "'hallo'"
+              return: !oracle
+                value: !expression "'hallo'"
     """
     json_str = translate_to_test_suite(yaml_str)
     suite = parse_test_suite(json_str)
@@ -660,8 +660,8 @@ def test_value_built_in_checks_explicit():
       contexts:
         - testcases:
             - expression: 'test()'
-              return:
-                value: "'hallo'"
+              return: !oracle
+                value: "hallo"
                 oracle: "builtin"
     """
     json_str = translate_to_test_suite(yaml_str)
@@ -686,13 +686,13 @@ def test_value_custom_checks_correct():
       contexts:
         - testcases:
             - expression: 'test()'
-              return:
-                value: "'hallo'"
+              return: !oracle
+                value: "hallo"
                 oracle: "custom_check"
                 language: "python"
                 file: "test.py"
                 name: "evaluate_test"
-                arguments: ["'yes'", "5", "set([5, 5])"]
+                arguments: ['yes', 5, !expression "set([5, 5])"]
     """
     json_str = translate_to_test_suite(yaml_str)
     suite = parse_test_suite(json_str)