Tweak exception handling in problem statements

tested/descriptions/renderer.py

@@ -1,7 +1,6 @@
 """
 A Marko renderer that only renders TESTed code; all other things are left alone.
 """
-from doctest import DocTestParser
 
 from marko import block
 from marko.md_renderer import MarkdownRenderer
@@ -10,9 +9,7 @@
 from tested.dsl import parse_dsl, parse_string
 from tested.judge.evaluation import Channel, guess_expected_value, should_show
 from tested.languages.generation import generate_statement, get_readable_input
-from tested.testsuite import OutputChannel, Testcase
-
-TESTED_EXAMPLE_FORMAT = "console?lang=tested"
+from tested.testsuite import ExceptionOutputChannel, OutputChannel, Testcase
 
 
 def render_one_statement(bundle: Bundle, statement: str) -> str:
@@ -31,6 +28,17 @@ def _add_output(
 ):
  if should_show(output, channel):
  expected = guess_expected_value(bundle, output)
+ # Special handling of exceptions
+ if channel == Channel.EXCEPTION:
+ assert isinstance(output, ExceptionOutputChannel)
+ if output.exception and not output.exception.get_type(
+ bundle.config.programming_language
+ ):
+ name = bundle.language.get_declaration_metadata().get(
+ "exception", "Exception"
+ )
+ expected = f"{name}: {expected}"
+
  results.append(expected)
 
 
@@ -48,35 +56,6 @@ def get_expected_output(bundle: Bundle, tc: Testcase) -> list[str]:
 
 class TestedRenderer(MarkdownRenderer):
  bundle: Bundle
- _doctest_parser: DocTestParser
-
- def __init__(self):
- super().__init__()
- self._doctest_parser = DocTestParser()
-
- def _render_doctest(self, element: block.FencedCode) -> str:
- """
- Render a "doctest" code block.
- """
- assert element.lang == TESTED_EXAMPLE_FORMAT
- raw_code = self.render_children(element)
- doctests = self._doctest_parser.get_examples(raw_code)
-
- resulting_lines = []
- prompt = self.bundle.language.get_declaration_metadata().get("prompt", ">")
-
- # Both the doctests and the results are parsed as values in the DSL.
- for examples in doctests:
- generated_statement = render_one_statement(self.bundle, examples.source)
- resulting_lines.append(f"{prompt} {generated_statement}")
- resulting_lines.append(examples.want.lstrip())
-
- language = (
- f"console?lang={self.bundle.config.programming_language}&prompt={prompt}"
- )
- body = "\n".join(resulting_lines)
-
- return f"```{language}\n{body}```\n"
 
  def _render_normal_statements(self, element: block.FencedCode) -> str:
  """
@@ -133,7 +112,5 @@ def render_fenced_code(self, element: block.FencedCode) -> str:
  return self._render_normal_statements(element)
  elif element.lang == "dsl":
  return self._render_dsl_statements(element)
- elif element.lang == TESTED_EXAMPLE_FORMAT:
- return self._render_doctest(element)
  else:
  return super().render_fenced_code(element)

tested/languages/config.py

@@ -41,6 +41,7 @@ class TypeDeclarationMetadata(TypedDict):
  nested_overrides: NotRequired[dict[AllTypes, tuple[str, str]]]
  prompt: NotRequired[str]
  natural_overrides: NotRequired[dict[str, dict[AllTypes, tuple[str, str]]]]
+ exception: NotRequired[str]
 
 
 class Language(ABC):

tested/languages/csharp/config.py

@@ -266,7 +266,8 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata:
  "list": "List",
  "tuple": "Tuple",
  "any": "Object",
- }
+ },
+ "exception": "Exception",
  }
 
  def is_void_method(self, name: str) -> bool:

tested/languages/haskell/config.py

@@ -209,4 +209,5 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata:
  "sequence": True,
  },
  "nested_overrides": {"tuple": ("(", ")")}, # type: ignore
+ "exception": "Exception",
  }

tested/languages/java/config.py

@@ -207,6 +207,7 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata:
  },
  "nested": ("<", ">"),
  "nested_overrides": {"array": ("[", "]")}, # type: ignore
+ "exception": "Exception",
  }
 
  def is_void_method(self, name: str) -> bool:

tested/languages/javascript/config.py

@@ -235,4 +235,5 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata:
  "any": "object",
  },
  "nested": ("<", ">"),
+ "exception": "Error",
  }

tested/languages/kotlin/config.py

@@ -262,4 +262,5 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata:
  "any": "Any",
  },
  "nested": ("<", ">"),
+ "exception": "Exception",
  }

tested/languages/python/config.py

@@ -262,4 +262,5 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata:
  "any": "Any",
  },
  "prompt": ">>>",
+ "exception": "Exception",
  }

tests/descriptions/example.haskell.md

@@ -6,7 +6,3 @@ aFunctionCall "yes"
 ```
 
 This is haskell.
-
-```console?lang=haskell&prompt=>
-> aFunctionCall "yes"
-```

tests/descriptions/example.java.md

@@ -6,7 +6,3 @@ Submission.aFunctionCall("yes")
 ```
 
 This is java.
-
-```console?lang=java&prompt=>
-> Submission.aFunctionCall("yes")
-```

tests/descriptions/example.kotlin.md

@@ -6,7 +6,3 @@ aFunctionCall("yes")
 ```
 
 This is kotlin.
-
-```console?lang=kotlin&prompt=>
-> aFunctionCall("yes")
-```

tests/descriptions/example.md.jinja2

@@ -15,6 +15,3 @@ This is java.
 This is haskell.
 {% endif %}
 
-```console?lang=tested
->>> a_function_call("yes")
-```

tests/descriptions/example.python.md

@@ -6,7 +6,3 @@ a_function_call('yes')
 ```
 
 This is python.
-
-```console?lang=python&prompt=>>>
->>> a_function_call('yes')
-```

tests/descriptions/recoupling.md.j2

@@ -1,14 +1,14 @@
 Write a function `{{ function('divide') }}` that takes two arguments:
 _i_) a word (`{{ datatype("text")}}`) and
 _ii_) the number of (non-overlapping) groups $$n \\in \\mathbb{N}\_0$$ (`{{datatype("integer")}}`) into which the word must be divided.
-If the word passed to the function `{{function('divide')}}` cannot be divided into $$n$$ groups that have the same length, an `AssertionError` must be raised with the message `invalid division`.
+If the word passed to the function `{{function('divide')}}` cannot be divided into $$n$$ groups that have the same length, an exception must be raised with the message `invalid division`.
 Otherwise, the function must return a {{datatype("list").singular}} (`{{datatype("list")}}`) containing the $$n$$ groups (`{{datatype("text")}}`) into which the given word can be divided.
 All groups need to have the same length (same number of letters).
 
 Write another function `{{function('recouple')}}` that takes two arguments:
 _i_) a {{datatype("sequence").singular}} (`{{datatype("sequence")}}`) of $$m \\in \\mathbb{N}\_0$$ words (`{{datatype("text")}}`) and
 _ii_) the number of (non-overlapping) groups $$n \\in \\mathbb{N}\_0$$ (`{{datatype("integer")}}`) into which the words must be divided.
-If at least one of the words passed to the function `{{function('recouple')}}` cannot be divided into $$n$$ groups that have the same length, an `AssertionError` must be raised with the message `invalid division`.
+If at least one of the words passed to the function `{{function('recouple')}}` cannot be divided into $$n$$ groups that have the same length, an exception must be raised with the message `invalid division`.
 Otherwise, the function must return a {{datatype("sequence").singular}} containing the $$n$$ new words (`{{datatype("text")}}`) obtained when each of the $$m$$ given words is divided into $$n$$ groups that have the same length, and if each of the $$m$$ corresponding groups is merged into a new word.
 The type of the returned {{datatype("sequence").singular}} (`{{datatype("sequence")}}`) must correspond to the type of the {{datatype("sequence").singular}} passed as a first argument to the function.
 
@@ -23,23 +23,13 @@ units:
  - expression: "divide('COMMUNED', 4)"
  return: ["CO", "MM", "UN", "ED"]
  - expression: "divide('programming', 5)"
- exception:
- message: "invalid division"
- types:
- python: AssertionError
- javascript: AssertionError
- java: IllegalArgumentException
+ exception: "invalid division"
 - unit: "Recouple"
  scripts:
  - expression: "recouple(['ACcoST', 'COmmIT', 'LAunCH', 'DEedED'], 3)"
  return: !list ["ACCOLADE", "communed", "STITCHED"]
  - expression: "recouple(('ACCOLADE', 'communed', 'STITCHED'), 4)"
  return: !tuple ["ACcoST", "COmmIT", "LAunCH", "DEedED"]
  - expression: "recouple(['programming', 'computer', 'games'], 5)"
- exception:
- message: "invalid division"
- types:
- python: AssertionError
- javascript: AssertionError
- java: IllegalArgumentException
+ exception: "invalid division"
 ```

tests/descriptions/recoupling.python.md

@@ -1,14 +1,14 @@
 Write a function `divide` that takes two arguments:
 *i*) a word (`str`) and
 *ii*) the number of (non-overlapping) groups $$n \\in \\mathbb{N}\_0$$ (`int`) into which the word must be divided.
-If the word passed to the function `divide` cannot be divided into $$n$$ groups that have the same length, an `AssertionError` must be raised with the message `invalid division`.
+If the word passed to the function `divide` cannot be divided into $$n$$ groups that have the same length, an exception must be raised with the message `invalid division`.
 Otherwise, the function must return a list (`list`) containing the $$n$$ groups (`str`) into which the given word can be divided.
 All groups need to have the same length (same number of letters).
 
 Write another function `recouple` that takes two arguments:
 *i*) a sequence (`list` or `tuple`) of $$m \\in \\mathbb{N}\_0$$ words (`str`) and
 *ii*) the number of (non-overlapping) groups $$n \\in \\mathbb{N}\_0$$ (`int`) into which the words must be divided.
-If at least one of the words passed to the function `recouple` cannot be divided into $$n$$ groups that have the same length, an `AssertionError` must be raised with the message `invalid division`.
+If at least one of the words passed to the function `recouple` cannot be divided into $$n$$ groups that have the same length, an exception must be raised with the message `invalid division`.
 Otherwise, the function must return a sequence containing the $$n$$ new words (`str`) obtained when each of the $$m$$ given words is divided into $$n$$ groups that have the same length, and if each of the $$m$$ corresponding groups is merged into a new word.
 The type of the returned sequence (`list` or `tuple`) must correspond to the type of the sequence passed as a first argument to the function.
 
@@ -20,12 +20,12 @@ The type of the returned sequence (`list` or `tuple`) must correspond to the typ
 >>> divide('COMMUNED', 4)
 ['CO', 'MM', 'UN', 'ED']
 >>> divide('programming', 5)
-AssertionError: invalid division
+Exception: invalid division
 
 >>> recouple(['ACcoST', 'COmmIT', 'LAunCH', 'DEedED'], 3)
 ['ACCOLADE', 'communed', 'STITCHED']
 >>> recouple(('ACCOLADE', 'communed', 'STITCHED'), 4)
 ('ACcoST', 'COmmIT', 'LAunCH', 'DEedED')
 >>> recouple(['programming', 'computer', 'games'], 5)
-AssertionError: invalid division
+Exception: invalid division
 ```

tests/test_problem_statements.py

@@ -8,7 +8,7 @@
 
 
 @pytest.mark.parametrize("language", ["python", "kotlin", "java", "haskell"])
-def test_python_description(language: str):
+def test_small_descriptions(language: str):
  test_dir = Path(__file__).parent
  description_template = test_dir / "descriptions" / "example.md.jinja2"
  description_python = test_dir / "descriptions" / f"example.{language}.md"
@@ -18,7 +18,7 @@ def test_python_description(language: str):
  expected = dp.read()
  actual = process_problem_statement(template, language)
 
- assert actual == expected
+ assert actual.strip() == expected.strip()
  assert f"This is {language}." in actual
 
 
@@ -130,38 +130,6 @@ def test_template_natural_type_name_nl(lang: str, tested_type: Any, expected: st
  assert instance == f"{expected}"
 
 
-@pytest.mark.parametrize(
- ("lang", "prompt"),
- [
- ("python", ">>>"),
- ("java", ">"),
- ("c", ">"),
- ("kotlin", ">"),
- ("javascript", ">"),
- ("haskell", ">"),
- ],
-)
-def test_template_code_block_markdown(lang: str, prompt: str):
- template = """```console?lang=tested
->>> random()
-5
-```"""
- expected_stmt = (
- "random "
- if lang == "haskell"
- else "Submission.random()"
- if lang == "java"
- else "random()"
- )
- expected_expr = "5"
- instance = process_problem_statement(template, lang)
- expected = f"""```console?lang={lang}&prompt={prompt}
-{prompt} {expected_stmt}
-{expected_expr}
-```"""
- assert instance == expected
-
-
 @pytest.mark.parametrize(
  ("lang", "expected"),
  [
@@ -227,62 +195,6 @@ def test_template_failed_string():
  process_problem_statement(template, "java")
 
 
-def test_multiline_results():
- template = """
-```console?lang=tested
->>> dots("paper.txt")
-###..###...##..#..#.####.###..#....###.
-#..#.#..#.#..#.#.#..#....#..#.#....#..#
-#..#.#..#.#....##...###..###..#....#..#
-###..###..#....#.#..#....#..#.#....###.
-#.#..#....#..#.#.#..#....#..#.#....#.#.
-#..#.#.....##..#..#.#....###..####.#..#
-```
-"""
- actual = process_problem_statement(template, "javascript")
- expected = """
-```console?lang=javascript&prompt=>
-> dots("paper.txt")
-###..###...##..#..#.####.###..#....###.
-#..#.#..#.#..#.#.#..#....#..#.#....#..#
-#..#.#..#.#....##...###..###..#....#..#
-###..###..#....#.#..#....#..#.#....###.
-#.#..#....#..#.#.#..#....#..#.#....#.#.
-#..#.#.....##..#..#.#....###..####.#..#
-```
-"""
- assert actual == expected
-
-
-def test_multiline_statement():
- template = """
-```console?lang=tested
->>> dots(
-... "paper.txt"
-... )
-###..###...##..#..#.####.###..#....###.
-#..#.#..#.#..#.#.#..#....#..#.#....#..#
-#..#.#..#.#....##...###..###..#....#..#
-###..###..#....#.#..#....#..#.#....###.
-#.#..#....#..#.#.#..#....#..#.#....#.#.
-#..#.#.....##..#..#.#....###..####.#..#
-```
-"""
- actual = process_problem_statement(template, "javascript")
- expected = """
-```console?lang=javascript&prompt=>
-> dots("paper.txt")
-###..###...##..#..#.####.###..#....###.
-#..#.#..#.#..#.#.#..#....#..#.#....#..#
-#..#.#..#.#....##...###..###..#....#..#
-###..###..#....#.#..#....#..#.#....###.
-#.#..#....#..#.#.#..#....#..#.#....#.#.
-#..#.#.....##..#..#.#....###..####.#..#
-```
-"""
- assert actual == expected
-
-
 def test_long_description():
  test_dir = Path(__file__).parent
  description_template = test_dir / "descriptions" / "recoupling.md.j2"