Merge pull request #491 from dodona-edu/enhance/problem-statements

Enhance problem statements

README.md

@@ -107,7 +107,7 @@ $ cat exercise/simple-example/config.json
   "source": "exercise/simple-example/correct.py",
   "judge": ".",
   "workdir": "workdir/",
-  "plan_name": "suite.yaml",
+  "test_suite": "suite.yaml",
   "memory_limit": 536870912,
   "time_limit": 60
 }
@@ -120,7 +120,7 @@ These attributes are used by TESTed:
 - `source`: path of the submission that must be evaluated
 - `judge`: path of the root directory of TESTEd
 - `workdir`: path of a temporary directory (see below)
-- `plan_name`: path of the test suite, relative to the resources directory (as defined above)
+- `test_suite`: path of the test suite, relative to the resources directory (as defined above)
 
 Before evaluating a submission, TESTed generates test code in the workdir.
 Create that directory:

tested/datatypes/__init__.py

@@ -64,6 +64,8 @@ def resolve_to_basic(type_: AllTypes) -> BasicTypes:
 def string_to_type(type_string: str) -> AllTypes:
     enums = get_args(AllTypes)
     for enum in enums:
-        if type_string in enum.__members__:
-            return enum[type_string]
+        try:
+            return enum(type_string)
+        except ValueError:
+            pass
     raise ValueError(f"Unknown type string {type_string}")

tested/datatypes/advanced.py

@@ -91,6 +91,10 @@ class AdvancedStringTypes(_AdvancedDataType):
     """
     A single character
     """
+    STRING = "string", BasicStringTypes.TEXT
+    """
+    A string (sequence of characters).
+    """
 
 
 class AdvancedNothingTypes(_AdvancedDataType):

tested/descriptions/converters.py

@@ -1,12 +1,14 @@
 from functools import partial
 from typing import cast
 
+from attr import dataclass
 from jinja2 import Template
 from marko import Markdown
 
 from tested.configs import Bundle
-from tested.datatypes import AllTypes
+from tested.datatypes import AdvancedTypes, AllTypes, string_to_type
 from tested.descriptions.renderer import TestedRenderer, render_one_statement
+from tested.features import TypeSupport
 from tested.internationalization import get_i18n_string, set_locale
 from tested.languages import Language
 from tested.languages.conventionalize import (
@@ -18,22 +20,74 @@
     conventionalize_property,
 )
 from tested.languages.generation import NestedTypeDeclaration, generate_type_declaration
+from tested.utils import get_args
 
 
-def type_declaration(
-    language: Language, type_: AllTypes, *others: NestedTypeDeclaration
-) -> str:
-    if len(others):
-        result = generate_type_declaration(language, (type_, others))
-    else:
-        result = generate_type_declaration(language, type_)
-    assert isinstance(result, str)
-    return result
+@dataclass
+class Datatype:
+    locale: str
+    language: Language
+    type_: AllTypes
+    others: tuple[NestedTypeDeclaration]
 
+    def _types(self) -> list[str]:
+        if len(self.others):
+            result = [
+                generate_type_declaration(self.language, (self.type_, self.others))
+            ]
+        elif isinstance(self.type_, AdvancedTypes):
+            result = [generate_type_declaration(self.language, self.type_)]
+        else:
+            possible_types = [self.type_]
+            supported_types = self.language.datatype_support()
+            for advanced_type_enum in get_args(AdvancedTypes):
+                for advanced_type in advanced_type_enum:
+                    if (
+                        advanced_type.base_type == self.type_
+                        and supported_types.get(advanced_type) == TypeSupport.SUPPORTED
+                    ):
+                        possible_types.append(advanced_type)
 
-def common_type_name(type_: AllTypes, plural: bool = False):
-    key = "plural" if plural else "singular"
-    return get_i18n_string(f"types.{key}.{type_}")
+            all_types = {
+                generate_type_declaration(self.language, x) for x in possible_types
+            }
+            result = sorted(all_types)
+
+        assert (
+            len(result) > 0
+        ), f"Could not find concrete type for {self.type_} in {self.language.__class__.__name__}"
+        return result
+
+    def __str__(self) -> str:
+        types = self._types()
+        types = [f"`{x}`" for x in types]
+        last_sep = f" {get_i18n_string('types.joiner')} "
+        return last_sep.join(
+            [", ".join(types[:-1]), types[-1]] if len(types) > 2 else types
+        )
+
+    @property
+    def simple(self) -> str:
+        if len(self.others):
+            return generate_type_declaration(self.language, (self.type_, self.others))
+        else:
+            return generate_type_declaration(self.language, self.type_)
+
+    @property
+    def singular(self) -> str:
+        return get_i18n_string(f"types.singular.{self.type_}")
+
+    @property
+    def plural(self) -> str:
+        return get_i18n_string(f"types.singular.{self.type_}")
+
+
+def construct_datatype(
+    locale: str, language: Language, type_: str, *others: NestedTypeDeclaration
+) -> Datatype:
+    enum_type = string_to_type(type_)
+    # noinspection PyTypeChecker
+    return Datatype(locale=locale, language=language, type_=enum_type, others=others)
 
 
 def convert_templated_problem(bundle: Bundle, raw_description: str) -> str:
@@ -61,8 +115,7 @@ def convert_templated_problem(bundle: Bundle, raw_description: str) -> str:
         # Access to the current programming language
         programming_language=bundle.config.programming_language,
         # Data type conversion
-        datatype=partial(type_declaration, language),
-        datatype_common=common_type_name,
+        datatype=partial(construct_datatype, bundle.config.natural_language, language),
         t=partial(render_one_statement, bundle),
     )
 

tested/descriptions/renderer.py

@@ -7,8 +7,10 @@
 from marko.md_renderer import MarkdownRenderer
 
 from tested.configs import Bundle
-from tested.dsl import parse_string
-from tested.languages.generation import generate_statement
+from tested.dsl import parse_dsl, parse_string
+from tested.judge.evaluation import Channel, guess_expected_value, should_show
+from tested.languages.generation import generate_statement, get_readable_input
+from tested.testsuite import OutputChannel, Testcase
 
 TESTED_EXAMPLE_FORMAT = "console?lang=tested"
 
@@ -23,6 +25,27 @@ def render_one_statement(bundle: Bundle, statement: str) -> str:
     return bundle.language.cleanup_description(generated_statement)
 
 
+# Similar to _add_channel
+def _add_output(
+    bundle: Bundle, output: OutputChannel, channel: Channel, results: list[str]
+):
+    if should_show(output, channel):
+        expected = guess_expected_value(bundle, output)
+        results.append(expected)
+
+
+def get_expected_output(bundle: Bundle, tc: Testcase) -> list[str]:
+    results = []
+    _add_output(bundle, tc.output.stdout, Channel.STDOUT, results)
+    _add_output(bundle, tc.output.stderr, Channel.STDERR, results)
+    _add_output(bundle, tc.output.file, Channel.FILE, results)
+    _add_output(bundle, tc.output.exception, Channel.EXCEPTION, results)
+    _add_output(bundle, tc.output.result, Channel.RETURN, results)
+    _add_output(bundle, tc.output.exit_code, Channel.EXIT, results)
+
+    return results
+
+
 class TestedRenderer(MarkdownRenderer):
     bundle: Bundle
     _doctest_parser: DocTestParser
@@ -72,9 +95,44 @@ def _render_normal_statements(self, element: block.FencedCode) -> str:
         body = "\n".join(resulting_lines)
         return f"```{language}\n{body}\n```\n"
 
+    def _render_dsl_statements(self, element: block.FencedCode) -> str:
+        """
+        Render a single statement (or multiple lines of single statements).
+        """
+        assert element.lang == "dsl"
+
+        rendered_dsl = self.render_children(element)
+
+        # Parse the DSL
+        parsed_dsl = parse_dsl(rendered_dsl)
+
+        # Get all actual tests
+        tests = []
+        for tab in parsed_dsl.tabs:
+            for context in tab.contexts:
+                for testcase in context.testcases:
+                    tests.append(testcase)
+
+        resulting_lines = []
+        prompt = self.bundle.language.get_declaration_metadata().get("prompt", ">")
+        for testcase in tests:
+            stmt_message, _ = get_readable_input(self.bundle, testcase)
+            resulting_lines.append(f"{prompt} {stmt_message.description}")
+            output_lines = get_expected_output(self.bundle, testcase)
+            resulting_lines.extend(output_lines)
+
+        language = (
+            f"console?lang={self.bundle.config.programming_language}&prompt={prompt}"
+        )
+        body = "\n".join(resulting_lines)
+
+        return f"```{language}\n{body}```\n"
+
     def render_fenced_code(self, element: block.FencedCode) -> str:
         if element.lang == "tested":
             return self._render_normal_statements(element)
+        elif element.lang == "dsl":
+            return self._render_dsl_statements(element)
         elif element.lang == TESTED_EXAMPLE_FORMAT:
             return self._render_doctest(element)
         else:

tested/internationalization/en.yaml

@@ -170,6 +170,7 @@ en:
       return: "Parse YAML return values"
       return-raw: "Parse return-raw values"
   types:
+    joiner: "or"
     singular:
       integer: "integer"
       real: "real number"
@@ -189,7 +190,7 @@ en:
       bigint: "big integer"
       single_precision: "single precision floating point number"
       double_precision: "double precision floating point number"
-      fixed_precision: "fixed precision number"  
+      fixed_precision: "fixed precision number"
       any: "any"
       list: "list"
       tuple: "tuple"

tested/internationalization/nl.yaml

@@ -170,6 +170,7 @@ nl:
       return: "Parsen van YAML returnwaarden"
       return-raw: "Parsen return-raw waarden"
   types:
+    joiner: "of"
     singular:
       integer: "geheel getal"
       real: "reëel getal"

tested/languages/bash/config.py

@@ -31,6 +31,7 @@ def naming_conventions(self) -> dict[Conventionable, NamingConventions]:
     def datatype_support(self) -> dict[AllTypes, TypeSupport]:
         return {
             AdvancedStringTypes.CHAR: TypeSupport.REDUCED,
+            AdvancedStringTypes.STRING: TypeSupport.SUPPORTED,
             BasicStringTypes.TEXT: TypeSupport.SUPPORTED,
         }
 
@@ -107,9 +108,6 @@ def generate_encoder(self, values: list[Value]) -> str:
 
     def get_declaration_metadata(self) -> TypeDeclarationMetadata:
         return {
-            "names": {  # type: ignore
-                "text": "str",
-                "char": "str",
-            },
+            "names": {"text": "str", "char": "str", "string": "str"},  # type: ignore
             "prompt": "$",
         }

tested/languages/c/config.py

@@ -54,6 +54,7 @@ def datatype_support(self) -> dict[AllTypes, TypeSupport]:
             "real": "supported",
             "char": "supported",
             "text": "supported",
+            "string": "supported",
             "boolean": "supported",
             "nothing": "supported",
             "undefined": "reduced",
@@ -166,6 +167,7 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata:
                 "real": "double",
                 "char": "char",
                 "text": "char*",
+                "string": "char*",
                 "boolean": "bool",
                 "nothing": "void",
                 "undefined": "void",

tested/languages/csharp/config.py

@@ -70,6 +70,7 @@ def datatype_support(self) -> dict[AllTypes, TypeSupport]:
             "integer": "supported",
             "real": "supported",
             "char": "reduced",
+            "string": "supported",
             "text": "supported",
             "boolean": "supported",
             "sequence": "supported",
@@ -245,6 +246,7 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata:
                 "real": "Double",
                 "char": "char",
                 "text": "string",
+                "string": "string",
                 "boolean": "Boolean",
                 "sequence": "List",
                 "set": "Set",

tested/languages/generation.py

@@ -91,7 +91,6 @@ def _handle_link_files(link_files: Iterable[FileUrl], language: str) -> tuple[st
 
 def _get_heredoc_token(stdin: str) -> str:
     delimiter = "STDIN"
-    stdin_lines = stdin.splitlines()
     while delimiter in stdin:
         delimiter = delimiter + "N"
     return delimiter
@@ -305,9 +304,13 @@ def _convert_single_type(
 
 def generate_type_declaration(
     language: Language, declaration: NestedTypeDeclaration, inner: bool = False
-) -> str | bool:
+) -> str:
     if not isinstance(declaration, tuple):
-        return _convert_single_type(language, declaration, inner)
+        simple_result = _convert_single_type(language, declaration, inner)
+        assert isinstance(
+            simple_result, str
+        ), f"{declaration} is a simple type and should generate a string"
+        return simple_result
 
     meta = language.get_declaration_metadata()
     base_type, nested = declaration
@@ -326,7 +329,6 @@ def generate_type_declaration(
     converted_nested = []
     for x in nested:
         converted_x = generate_type_declaration(language, x, True)
-        assert isinstance(converted_x, str)
         converted_nested.append(converted_x)
 
     if base is True:

tested/languages/haskell/config.py

@@ -50,6 +50,7 @@ def datatype_support(self) -> dict[AllTypes, TypeSupport]:
             "integer": "supported",
             "real": "supported",
             "char": "supported",
+            "string": "supported",
             "text": "supported",
             "boolean": "supported",
             "sequence": "supported",
@@ -186,6 +187,7 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata:
                 "integer": "Int",
                 "real": "Double",
                 "char": "Char",
+                "string": "String",
                 "text": "String",
                 "boolean": "Bool",
                 "nothing": "Nothing",

tested/languages/java/config.py

@@ -65,6 +65,7 @@ def datatype_support(self) -> dict[AllTypes, TypeSupport]:
             "integer": "supported",
             "real": "supported",
             "char": "supported",
+            "string": "supported",
             "text": "supported",
             "boolean": "supported",
             "sequence": "supported",
@@ -165,6 +166,7 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata:
                 "real": "double",
                 "char": "char",
                 "text": "String",
+                "string": "String",
                 "boolean": "boolean",
                 "sequence": "List",
                 "set": "Set",

tested/languages/javascript/config.py

@@ -67,6 +67,7 @@ def datatype_support(self) -> dict[AllTypes, TypeSupport]:
             "real": "supported",
             "char": "reduced",
             "text": "supported",
+            "string": "supported",
             "boolean": "supported",
             "sequence": "supported",
             "set": "supported",
@@ -209,6 +210,7 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata:
                 "real": "number",
                 "char": "string",
                 "text": "string",
+                "string": "string",
                 "boolean": "boolean",
                 "sequence": "array",
                 "set": "set",