Enhance problem statements

README.md

@@ -107,7 +107,7 @@ $ cat exercise/simple-example/config.json
  "source": "exercise/simple-example/correct.py",
  "judge": ".",
  "workdir": "workdir/",
- "plan_name": "suite.yaml",
+ "test_suite": "suite.yaml",
  "memory_limit": 536870912,
  "time_limit": 60
 }
@@ -120,7 +120,7 @@ These attributes are used by TESTed:
 - `source`: path of the submission that must be evaluated
 - `judge`: path of the root directory of TESTEd
 - `workdir`: path of a temporary directory (see below)
-- `plan_name`: path of the test suite, relative to the resources directory (as defined above)
+- `test_suite`: path of the test suite, relative to the resources directory (as defined above)
 
 Before evaluating a submission, TESTed generates test code in the workdir.
 Create that directory:

tested/datatypes/__init__.py

@@ -64,6 +64,8 @@ def resolve_to_basic(type_: AllTypes) -> BasicTypes:
 def string_to_type(type_string: str) -> AllTypes:
  enums = get_args(AllTypes)
  for enum in enums:
- if type_string in enum.__members__:
- return enum[type_string]
+ try:
+ return enum(type_string)
+ except ValueError:
+ pass
  raise ValueError(f"Unknown type string {type_string}")

tested/datatypes/advanced.py

@@ -91,6 +91,10 @@ class AdvancedStringTypes(_AdvancedDataType):
  """
  A single character
  """
+ STRING = "string", BasicStringTypes.TEXT
+ """
+ A string (sequence of characters).
+ """
 
 
 class AdvancedNothingTypes(_AdvancedDataType):

tested/descriptions/converters.py

@@ -1,12 +1,14 @@
 from functools import partial
 from typing import cast
 
+from attr import dataclass
 from jinja2 import Template
 from marko import Markdown
 
 from tested.configs import Bundle
-from tested.datatypes import AllTypes
+from tested.datatypes import AdvancedTypes, AllTypes, string_to_type
 from tested.descriptions.renderer import TestedRenderer, render_one_statement
+from tested.features import TypeSupport
 from tested.internationalization import get_i18n_string, set_locale
 from tested.languages import Language
 from tested.languages.conventionalize import (
@@ -18,22 +20,74 @@
  conventionalize_property,
 )
 from tested.languages.generation import NestedTypeDeclaration, generate_type_declaration
+from tested.utils import get_args
 
 
-def type_declaration(
- language: Language, type_: AllTypes, *others: NestedTypeDeclaration
-) -> str:
- if len(others):
- result = generate_type_declaration(language, (type_, others))
- else:
- result = generate_type_declaration(language, type_)
- assert isinstance(result, str)
- return result
+@dataclass
+class Datatype:
+ locale: str
+ language: Language
+ type_: AllTypes
+ others: tuple[NestedTypeDeclaration]
 
+ def _types(self) -> list[str]:
+ if len(self.others):
+ result = [
+ generate_type_declaration(self.language, (self.type_, self.others))
+ ]
+ elif isinstance(self.type_, AdvancedTypes):
+ result = [generate_type_declaration(self.language, self.type_)]
+ else:
+ possible_types = [self.type_]
+ supported_types = self.language.datatype_support()
+ for advanced_type_enum in get_args(AdvancedTypes):
+ for advanced_type in advanced_type_enum:
+ if (
+ advanced_type.base_type == self.type_
+ and supported_types.get(advanced_type) == TypeSupport.SUPPORTED
+ ):
+ possible_types.append(advanced_type)
 
-def common_type_name(type_: AllTypes, plural: bool = False):
- key = "plural" if plural else "singular"
- return get_i18n_string(f"types.{key}.{type_}")
+ all_types = {
+ generate_type_declaration(self.language, x) for x in possible_types
+ }
+ result = sorted(all_types)
+
+ assert (
+ len(result) > 0
+ ), f"Could not find concrete type for {self.type_} in {self.language.__class__.__name__}"
+ return result
+
+ def __str__(self) -> str:
+ types = self._types()
+ types = [f"`{x}`" for x in types]
+ last_sep = f" {get_i18n_string('types.joiner')} "
+ return last_sep.join(
+ [", ".join(types[:-1]), types[-1]] if len(types) > 2 else types
+ )
+
+ @property
+ def simple(self) -> str:
+ if len(self.others):
+ return generate_type_declaration(self.language, (self.type_, self.others))
+ else:
+ return generate_type_declaration(self.language, self.type_)
+
+ @property
+ def singular(self) -> str:
+ return get_i18n_string(f"types.singular.{self.type_}")
+
+ @property
+ def plural(self) -> str:
+ return get_i18n_string(f"types.singular.{self.type_}")
+
+
+def construct_datatype(
+ locale: str, language: Language, type_: str, *others: NestedTypeDeclaration
+) -> Datatype:
+ enum_type = string_to_type(type_)
+ # noinspection PyTypeChecker
+ return Datatype(locale=locale, language=language, type_=enum_type, others=others)
 
 
 def convert_templated_problem(bundle: Bundle, raw_description: str) -> str:
@@ -61,8 +115,7 @@
  # Access to the current programming language
  programming_language=bundle.config.programming_language,
  # Data type conversion
- datatype=partial(type_declaration, language),
- datatype_common=common_type_name,
+ datatype=partial(construct_datatype, bundle.config.natural_language, language),
  t=partial(render_one_statement, bundle),
  )
 

tested/descriptions/renderer.py

@@ -7,8 +7,10 @@
 from marko.md_renderer import MarkdownRenderer
 
 from tested.configs import Bundle
-from tested.dsl import parse_string
-from tested.languages.generation import generate_statement
+from tested.dsl import parse_dsl, parse_string
+from tested.judge.evaluation import Channel, guess_expected_value, should_show
+from tested.languages.generation import generate_statement, get_readable_input
+from tested.testsuite import OutputChannel, Testcase
 
 TESTED_EXAMPLE_FORMAT = "console?lang=tested"
 
@@ -23,6 +25,27 @@ def render_one_statement(bundle: Bundle, statement: str) -> str:
  return bundle.language.cleanup_description(generated_statement)
 
 
+# Similar to _add_channel
+def _add_output(
+ bundle: Bundle, output: OutputChannel, channel: Channel, results: list[str]
+):
+ if should_show(output, channel):
+ expected = guess_expected_value(bundle, output)
+ results.append(expected)
+
+
+def get_expected_output(bundle: Bundle, tc: Testcase) -> list[str]:
+ results = []
+ _add_output(bundle, tc.output.stdout, Channel.STDOUT, results)
+ _add_output(bundle, tc.output.stderr, Channel.STDERR, results)
+ _add_output(bundle, tc.output.file, Channel.FILE, results)
+ _add_output(bundle, tc.output.exception, Channel.EXCEPTION, results)
+ _add_output(bundle, tc.output.result, Channel.RETURN, results)
+ _add_output(bundle, tc.output.exit_code, Channel.EXIT, results)
+
+ return results
+
+
 class TestedRenderer(MarkdownRenderer):
  bundle: Bundle
  _doctest_parser: DocTestParser
@@ -72,9 +95,44 @@ def _render_normal_statements(self, element: block.FencedCode) -> str:
  body = "\n".join(resulting_lines)
  return f"```{language}\n{body}\n```\n"
 
+ def _render_dsl_statements(self, element: block.FencedCode) -> str:
+ """
+ Render a single statement (or multiple lines of single statements).
+ """
+ assert element.lang == "dsl"
+
+ rendered_dsl = self.render_children(element)
+
+ # Parse the DSL
+ parsed_dsl = parse_dsl(rendered_dsl)
+
+ # Get all actual tests
+ tests = []
+ for tab in parsed_dsl.tabs:
+ for context in tab.contexts:
+ for testcase in context.testcases:
+ tests.append(testcase)
+
+ resulting_lines = []
+ prompt = self.bundle.language.get_declaration_metadata().get("prompt", ">")
+ for testcase in tests:
+ stmt_message, _ = get_readable_input(self.bundle, testcase)
+ resulting_lines.append(f"{prompt} {stmt_message.description}")
+ output_lines = get_expected_output(self.bundle, testcase)
+ resulting_lines.extend(output_lines)
+
+ language = (
+ f"console?lang={self.bundle.config.programming_language}&prompt={prompt}"
+ )
+ body = "\n".join(resulting_lines)
+
+ return f"```{language}\n{body}```\n"
+
  def render_fenced_code(self, element: block.FencedCode) -> str:
  if element.lang == "tested":
  return self._render_normal_statements(element)
+ elif element.lang == "dsl":
+ return self._render_dsl_statements(element)
  elif element.lang == TESTED_EXAMPLE_FORMAT:
  return self._render_doctest(element)
  else:

tested/internationalization/en.yaml

@@ -170,6 +170,7 @@ en:
  return: "Parse YAML return values"
  return-raw: "Parse return-raw values"
  types:
+ joiner: "or"
  singular:
  integer: "integer"
  real: "real number"
@@ -189,7 +190,7 @@ en:
  bigint: "big integer"
  single_precision: "single precision floating point number"
  double_precision: "double precision floating point number"
- fixed_precision: "fixed precision number" 
+ fixed_precision: "fixed precision number"
  any: "any"
  list: "list"
  tuple: "tuple"

tested/internationalization/nl.yaml

@@ -170,6 +170,7 @@ nl:
  return: "Parsen van YAML returnwaarden"
  return-raw: "Parsen return-raw waarden"
  types:
+ joiner: "of"
  singular:
  integer: "geheel getal"
  real: "reëel getal"

tested/languages/bash/config.py

@@ -31,6 +31,7 @@ def naming_conventions(self) -> dict[Conventionable, NamingConventions]:
  def datatype_support(self) -> dict[AllTypes, TypeSupport]:
  return {
  AdvancedStringTypes.CHAR: TypeSupport.REDUCED,
+ AdvancedStringTypes.STRING: TypeSupport.SUPPORTED,
  BasicStringTypes.TEXT: TypeSupport.SUPPORTED,
  }
 
@@ -107,9 +108,6 @@ def generate_encoder(self, values: list[Value]) -> str:
 
  def get_declaration_metadata(self) -> TypeDeclarationMetadata:
  return {
- "names": { # type: ignore
- "text": "str",
- "char": "str",
- },
+ "names": {"text": "str", "char": "str", "string": "str"}, # type: ignore
  "prompt": "$",
  }

tested/languages/c/config.py

@@ -54,6 +54,7 @@ def datatype_support(self) -> dict[AllTypes, TypeSupport]:
  "real": "supported",
  "char": "supported",
  "text": "supported",
+ "string": "supported",
  "boolean": "supported",
  "nothing": "supported",
  "undefined": "reduced",
@@ -166,6 +167,7 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata:
  "real": "double",
  "char": "char",
  "text": "char*",
+ "string": "char*",
  "boolean": "bool",
  "nothing": "void",
  "undefined": "void",

tested/languages/csharp/config.py

@@ -70,6 +70,7 @@ def datatype_support(self) -> dict[AllTypes, TypeSupport]:
  "integer": "supported",
  "real": "supported",
  "char": "reduced",
+ "string": "supported",
  "text": "supported",
  "boolean": "supported",
  "sequence": "supported",
@@ -245,6 +246,7 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata:
  "real": "Double",
  "char": "char",
  "text": "string",
+ "string": "string",
  "boolean": "Boolean",
  "sequence": "List",
  "set": "Set",

tested/languages/generation.py

@@ -91,7 +91,6 @@ def _handle_link_files(link_files: Iterable[FileUrl], language: str) -> tuple[st
 
 def _get_heredoc_token(stdin: str) -> str:
  delimiter = "STDIN"
- stdin_lines = stdin.splitlines()
  while delimiter in stdin:
  delimiter = delimiter + "N"
  return delimiter
@@ -305,9 +304,13 @@ def _convert_single_type(
 
 def generate_type_declaration(
  language: Language, declaration: NestedTypeDeclaration, inner: bool = False
-) -> str | bool:
+) -> str:
  if not isinstance(declaration, tuple):
- return _convert_single_type(language, declaration, inner)
+ simple_result = _convert_single_type(language, declaration, inner)
+ assert isinstance(
+ simple_result, str
+ ), f"{declaration} is a simple type and should generate a string"
+ return simple_result
 
  meta = language.get_declaration_metadata()
  base_type, nested = declaration
@@ -326,7 +329,6 @@ def generate_type_declaration(
  converted_nested = []
  for x in nested:
  converted_x = generate_type_declaration(language, x, True)
- assert isinstance(converted_x, str)
  converted_nested.append(converted_x)
 
  if base is True:

tested/languages/haskell/config.py

@@ -50,6 +50,7 @@ def datatype_support(self) -> dict[AllTypes, TypeSupport]:
  "integer": "supported",
  "real": "supported",
  "char": "supported",
+ "string": "supported",
  "text": "supported",
  "boolean": "supported",
  "sequence": "supported",
@@ -186,6 +187,7 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata:
  "integer": "Int",
  "real": "Double",
  "char": "Char",
+ "string": "String",
  "text": "String",
  "boolean": "Bool",
  "nothing": "Nothing",

tested/languages/java/config.py

@@ -65,6 +65,7 @@ def datatype_support(self) -> dict[AllTypes, TypeSupport]:
  "integer": "supported",
  "real": "supported",
  "char": "supported",
+ "string": "supported",
  "text": "supported",
  "boolean": "supported",
  "sequence": "supported",
@@ -165,6 +166,7 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata:
  "real": "double",
  "char": "char",
  "text": "String",
+ "string": "String",
  "boolean": "boolean",
  "sequence": "List",
  "set": "Set",

tested/languages/javascript/config.py

@@ -67,6 +67,7 @@ def datatype_support(self) -> dict[AllTypes, TypeSupport]:
  "real": "supported",
  "char": "reduced",
  "text": "supported",
+ "string": "supported",
  "boolean": "supported",
  "sequence": "supported",
  "set": "supported",
@@ -209,6 +210,7 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata:
  "real": "number",
  "char": "string",
  "text": "string",
+ "string": "string",
  "boolean": "boolean",
  "sequence": "array",
  "set": "set",