From e0e4d5f8be711bcf1afc1e82465513ab2e37de95 Mon Sep 17 00:00:00 2001 From: Niko Strijbol Date: Mon, 29 Jan 2024 11:26:20 +0100 Subject: [PATCH] Tweak exception handling and remove tested doctest We no longer support the doctest format --- tested/descriptions/renderer.py | 47 ++++--------- tested/languages/config.py | 1 + tested/languages/csharp/config.py | 3 +- tested/languages/haskell/config.py | 1 + tested/languages/java/config.py | 1 + tested/languages/javascript/config.py | 1 + tested/languages/kotlin/config.py | 1 + tested/languages/python/config.py | 1 + tests/descriptions/example.haskell.md | 4 -- tests/descriptions/example.java.md | 4 -- tests/descriptions/example.kotlin.md | 4 -- tests/descriptions/example.md.jinja2 | 3 - tests/descriptions/example.python.md | 4 -- tests/descriptions/recoupling.md.j2 | 18 ++--- tests/descriptions/recoupling.python.md | 8 +-- tests/test_problem_statements.py | 92 +------------------------ 16 files changed, 30 insertions(+), 163 deletions(-) diff --git a/tested/descriptions/renderer.py b/tested/descriptions/renderer.py index 2e8ee30c..5690f12c 100644 --- a/tested/descriptions/renderer.py +++ b/tested/descriptions/renderer.py @@ -1,7 +1,6 @@ """ A Marko renderer that only renders TESTed code; all other things are left alone. """ -from doctest import DocTestParser from marko import block from marko.md_renderer import MarkdownRenderer @@ -10,9 +9,7 @@ from tested.dsl import parse_dsl, parse_string from tested.judge.evaluation import Channel, guess_expected_value, should_show from tested.languages.generation import generate_statement, get_readable_input -from tested.testsuite import OutputChannel, Testcase - -TESTED_EXAMPLE_FORMAT = "console?lang=tested" +from tested.testsuite import ExceptionOutputChannel, OutputChannel, Testcase def render_one_statement(bundle: Bundle, statement: str) -> str: @@ -31,6 +28,17 @@ def _add_output( ): if should_show(output, channel): expected = guess_expected_value(bundle, output) + # Special handling of exceptions + if channel == Channel.EXCEPTION: + assert isinstance(output, ExceptionOutputChannel) + if output.exception and not output.exception.get_type( + bundle.config.programming_language + ): + name = bundle.language.get_declaration_metadata().get( + "exception", "Exception" + ) + expected = f"{name}: {expected}" + results.append(expected) @@ -48,35 +56,6 @@ def get_expected_output(bundle: Bundle, tc: Testcase) -> list[str]: class TestedRenderer(MarkdownRenderer): bundle: Bundle - _doctest_parser: DocTestParser - - def __init__(self): - super().__init__() - self._doctest_parser = DocTestParser() - - def _render_doctest(self, element: block.FencedCode) -> str: - """ - Render a "doctest" code block. - """ - assert element.lang == TESTED_EXAMPLE_FORMAT - raw_code = self.render_children(element) - doctests = self._doctest_parser.get_examples(raw_code) - - resulting_lines = [] - prompt = self.bundle.language.get_declaration_metadata().get("prompt", ">") - - # Both the doctests and the results are parsed as values in the DSL. - for examples in doctests: - generated_statement = render_one_statement(self.bundle, examples.source) - resulting_lines.append(f"{prompt} {generated_statement}") - resulting_lines.append(examples.want.lstrip()) - - language = ( - f"console?lang={self.bundle.config.programming_language}&prompt={prompt}" - ) - body = "\n".join(resulting_lines) - - return f"```{language}\n{body}```\n" def _render_normal_statements(self, element: block.FencedCode) -> str: """ @@ -133,7 +112,5 @@ def render_fenced_code(self, element: block.FencedCode) -> str: return self._render_normal_statements(element) elif element.lang == "dsl": return self._render_dsl_statements(element) - elif element.lang == TESTED_EXAMPLE_FORMAT: - return self._render_doctest(element) else: return super().render_fenced_code(element) diff --git a/tested/languages/config.py b/tested/languages/config.py index 1fcd4ff4..a6320ab5 100644 --- a/tested/languages/config.py +++ b/tested/languages/config.py @@ -41,6 +41,7 @@ class TypeDeclarationMetadata(TypedDict): nested_overrides: NotRequired[dict[AllTypes, tuple[str, str]]] prompt: NotRequired[str] natural_overrides: NotRequired[dict[str, dict[AllTypes, tuple[str, str]]]] + exception: NotRequired[str] class Language(ABC): diff --git a/tested/languages/csharp/config.py b/tested/languages/csharp/config.py index f0667b0f..651d2ed0 100644 --- a/tested/languages/csharp/config.py +++ b/tested/languages/csharp/config.py @@ -266,7 +266,8 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata: "list": "List", "tuple": "Tuple", "any": "Object", - } + }, + "exception": "Exception", } def is_void_method(self, name: str) -> bool: diff --git a/tested/languages/haskell/config.py b/tested/languages/haskell/config.py index d4a81f83..7ae4cb82 100644 --- a/tested/languages/haskell/config.py +++ b/tested/languages/haskell/config.py @@ -209,4 +209,5 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata: "sequence": True, }, "nested_overrides": {"tuple": ("(", ")")}, # type: ignore + "exception": "Exception", } diff --git a/tested/languages/java/config.py b/tested/languages/java/config.py index 14532acc..67a7d716 100644 --- a/tested/languages/java/config.py +++ b/tested/languages/java/config.py @@ -207,6 +207,7 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata: }, "nested": ("<", ">"), "nested_overrides": {"array": ("[", "]")}, # type: ignore + "exception": "Exception", } def is_void_method(self, name: str) -> bool: diff --git a/tested/languages/javascript/config.py b/tested/languages/javascript/config.py index 94afd381..624374d5 100644 --- a/tested/languages/javascript/config.py +++ b/tested/languages/javascript/config.py @@ -235,4 +235,5 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata: "any": "object", }, "nested": ("<", ">"), + "exception": "Error", } diff --git a/tested/languages/kotlin/config.py b/tested/languages/kotlin/config.py index c1a294b4..c3467605 100644 --- a/tested/languages/kotlin/config.py +++ b/tested/languages/kotlin/config.py @@ -262,4 +262,5 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata: "any": "Any", }, "nested": ("<", ">"), + "exception": "Exception", } diff --git a/tested/languages/python/config.py b/tested/languages/python/config.py index 487a59f5..91138f0a 100644 --- a/tested/languages/python/config.py +++ b/tested/languages/python/config.py @@ -262,4 +262,5 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata: "any": "Any", }, "prompt": ">>>", + "exception": "Exception", } diff --git a/tests/descriptions/example.haskell.md b/tests/descriptions/example.haskell.md index 733db183..5bedd92a 100644 --- a/tests/descriptions/example.haskell.md +++ b/tests/descriptions/example.haskell.md @@ -6,7 +6,3 @@ aFunctionCall "yes" ``` This is haskell. - -```console?lang=haskell&prompt=> -> aFunctionCall "yes" -``` diff --git a/tests/descriptions/example.java.md b/tests/descriptions/example.java.md index f2a3c0ee..6b25bd9c 100644 --- a/tests/descriptions/example.java.md +++ b/tests/descriptions/example.java.md @@ -6,7 +6,3 @@ Submission.aFunctionCall("yes") ``` This is java. - -```console?lang=java&prompt=> -> Submission.aFunctionCall("yes") -``` diff --git a/tests/descriptions/example.kotlin.md b/tests/descriptions/example.kotlin.md index 111d5d42..892ffda8 100644 --- a/tests/descriptions/example.kotlin.md +++ b/tests/descriptions/example.kotlin.md @@ -6,7 +6,3 @@ aFunctionCall("yes") ``` This is kotlin. - -```console?lang=kotlin&prompt=> -> aFunctionCall("yes") -``` diff --git a/tests/descriptions/example.md.jinja2 b/tests/descriptions/example.md.jinja2 index 4a2c11d7..c9a81124 100644 --- a/tests/descriptions/example.md.jinja2 +++ b/tests/descriptions/example.md.jinja2 @@ -15,6 +15,3 @@ This is java. This is haskell. {% endif %} -```console?lang=tested ->>> a_function_call("yes") -``` diff --git a/tests/descriptions/example.python.md b/tests/descriptions/example.python.md index e46587a9..1211af48 100644 --- a/tests/descriptions/example.python.md +++ b/tests/descriptions/example.python.md @@ -6,7 +6,3 @@ a_function_call('yes') ``` This is python. - -```console?lang=python&prompt=>>> ->>> a_function_call('yes') -``` diff --git a/tests/descriptions/recoupling.md.j2 b/tests/descriptions/recoupling.md.j2 index a582acdc..0fca0f72 100644 --- a/tests/descriptions/recoupling.md.j2 +++ b/tests/descriptions/recoupling.md.j2 @@ -1,14 +1,14 @@ Write a function `{{ function('divide') }}` that takes two arguments: _i_) a word (`{{ datatype("text")}}`) and _ii_) the number of (non-overlapping) groups $$n \\in \\mathbb{N}\_0$$ (`{{datatype("integer")}}`) into which the word must be divided. -If the word passed to the function `{{function('divide')}}` cannot be divided into $$n$$ groups that have the same length, an `AssertionError` must be raised with the message `invalid division`. +If the word passed to the function `{{function('divide')}}` cannot be divided into $$n$$ groups that have the same length, an exception must be raised with the message `invalid division`. Otherwise, the function must return a {{datatype("list").singular}} (`{{datatype("list")}}`) containing the $$n$$ groups (`{{datatype("text")}}`) into which the given word can be divided. All groups need to have the same length (same number of letters). Write another function `{{function('recouple')}}` that takes two arguments: _i_) a {{datatype("sequence").singular}} (`{{datatype("sequence")}}`) of $$m \\in \\mathbb{N}\_0$$ words (`{{datatype("text")}}`) and _ii_) the number of (non-overlapping) groups $$n \\in \\mathbb{N}\_0$$ (`{{datatype("integer")}}`) into which the words must be divided. -If at least one of the words passed to the function `{{function('recouple')}}` cannot be divided into $$n$$ groups that have the same length, an `AssertionError` must be raised with the message `invalid division`. +If at least one of the words passed to the function `{{function('recouple')}}` cannot be divided into $$n$$ groups that have the same length, an exception must be raised with the message `invalid division`. Otherwise, the function must return a {{datatype("sequence").singular}} containing the $$n$$ new words (`{{datatype("text")}}`) obtained when each of the $$m$$ given words is divided into $$n$$ groups that have the same length, and if each of the $$m$$ corresponding groups is merged into a new word. The type of the returned {{datatype("sequence").singular}} (`{{datatype("sequence")}}`) must correspond to the type of the {{datatype("sequence").singular}} passed as a first argument to the function. @@ -23,12 +23,7 @@ units: - expression: "divide('COMMUNED', 4)" return: ["CO", "MM", "UN", "ED"] - expression: "divide('programming', 5)" - exception: - message: "invalid division" - types: - python: AssertionError - javascript: AssertionError - java: IllegalArgumentException + exception: "invalid division" - unit: "Recouple" scripts: - expression: "recouple(['ACcoST', 'COmmIT', 'LAunCH', 'DEedED'], 3)" @@ -36,10 +31,5 @@ units: - expression: "recouple(('ACCOLADE', 'communed', 'STITCHED'), 4)" return: !tuple ["ACcoST", "COmmIT", "LAunCH", "DEedED"] - expression: "recouple(['programming', 'computer', 'games'], 5)" - exception: - message: "invalid division" - types: - python: AssertionError - javascript: AssertionError - java: IllegalArgumentException + exception: "invalid division" ``` diff --git a/tests/descriptions/recoupling.python.md b/tests/descriptions/recoupling.python.md index e8da3717..98dc12db 100644 --- a/tests/descriptions/recoupling.python.md +++ b/tests/descriptions/recoupling.python.md @@ -1,14 +1,14 @@ Write a function `divide` that takes two arguments: *i*) a word (`str`) and *ii*) the number of (non-overlapping) groups $$n \\in \\mathbb{N}\_0$$ (`int`) into which the word must be divided. -If the word passed to the function `divide` cannot be divided into $$n$$ groups that have the same length, an `AssertionError` must be raised with the message `invalid division`. +If the word passed to the function `divide` cannot be divided into $$n$$ groups that have the same length, an exception must be raised with the message `invalid division`. Otherwise, the function must return a list (`list`) containing the $$n$$ groups (`str`) into which the given word can be divided. All groups need to have the same length (same number of letters). Write another function `recouple` that takes two arguments: *i*) a sequence (`list` or `tuple`) of $$m \\in \\mathbb{N}\_0$$ words (`str`) and *ii*) the number of (non-overlapping) groups $$n \\in \\mathbb{N}\_0$$ (`int`) into which the words must be divided. -If at least one of the words passed to the function `recouple` cannot be divided into $$n$$ groups that have the same length, an `AssertionError` must be raised with the message `invalid division`. +If at least one of the words passed to the function `recouple` cannot be divided into $$n$$ groups that have the same length, an exception must be raised with the message `invalid division`. Otherwise, the function must return a sequence containing the $$n$$ new words (`str`) obtained when each of the $$m$$ given words is divided into $$n$$ groups that have the same length, and if each of the $$m$$ corresponding groups is merged into a new word. The type of the returned sequence (`list` or `tuple`) must correspond to the type of the sequence passed as a first argument to the function. @@ -20,12 +20,12 @@ The type of the returned sequence (`list` or `tuple`) must correspond to the typ >>> divide('COMMUNED', 4) ['CO', 'MM', 'UN', 'ED'] >>> divide('programming', 5) -AssertionError: invalid division +Exception: invalid division >>> recouple(['ACcoST', 'COmmIT', 'LAunCH', 'DEedED'], 3) ['ACCOLADE', 'communed', 'STITCHED'] >>> recouple(('ACCOLADE', 'communed', 'STITCHED'), 4) ('ACcoST', 'COmmIT', 'LAunCH', 'DEedED') >>> recouple(['programming', 'computer', 'games'], 5) -AssertionError: invalid division +Exception: invalid division ``` diff --git a/tests/test_problem_statements.py b/tests/test_problem_statements.py index 89af0e08..b0979402 100644 --- a/tests/test_problem_statements.py +++ b/tests/test_problem_statements.py @@ -8,7 +8,7 @@ @pytest.mark.parametrize("language", ["python", "kotlin", "java", "haskell"]) -def test_python_description(language: str): +def test_small_descriptions(language: str): test_dir = Path(__file__).parent description_template = test_dir / "descriptions" / "example.md.jinja2" description_python = test_dir / "descriptions" / f"example.{language}.md" @@ -18,7 +18,7 @@ def test_python_description(language: str): expected = dp.read() actual = process_problem_statement(template, language) - assert actual == expected + assert actual.strip() == expected.strip() assert f"This is {language}." in actual @@ -130,38 +130,6 @@ def test_template_natural_type_name_nl(lang: str, tested_type: Any, expected: st assert instance == f"{expected}" -@pytest.mark.parametrize( - ("lang", "prompt"), - [ - ("python", ">>>"), - ("java", ">"), - ("c", ">"), - ("kotlin", ">"), - ("javascript", ">"), - ("haskell", ">"), - ], -) -def test_template_code_block_markdown(lang: str, prompt: str): - template = """```console?lang=tested ->>> random() -5 -```""" - expected_stmt = ( - "random " - if lang == "haskell" - else "Submission.random()" - if lang == "java" - else "random()" - ) - expected_expr = "5" - instance = process_problem_statement(template, lang) - expected = f"""```console?lang={lang}&prompt={prompt} -{prompt} {expected_stmt} -{expected_expr} -```""" - assert instance == expected - - @pytest.mark.parametrize( ("lang", "expected"), [ @@ -227,62 +195,6 @@ def test_template_failed_string(): process_problem_statement(template, "java") -def test_multiline_results(): - template = """ -```console?lang=tested ->>> dots("paper.txt") -###..###...##..#..#.####.###..#....###. -#..#.#..#.#..#.#.#..#....#..#.#....#..# -#..#.#..#.#....##...###..###..#....#..# -###..###..#....#.#..#....#..#.#....###. -#.#..#....#..#.#.#..#....#..#.#....#.#. -#..#.#.....##..#..#.#....###..####.#..# -``` -""" - actual = process_problem_statement(template, "javascript") - expected = """ -```console?lang=javascript&prompt=> -> dots("paper.txt") -###..###...##..#..#.####.###..#....###. -#..#.#..#.#..#.#.#..#....#..#.#....#..# -#..#.#..#.#....##...###..###..#....#..# -###..###..#....#.#..#....#..#.#....###. -#.#..#....#..#.#.#..#....#..#.#....#.#. -#..#.#.....##..#..#.#....###..####.#..# -``` -""" - assert actual == expected - - -def test_multiline_statement(): - template = """ -```console?lang=tested ->>> dots( -... "paper.txt" -... ) -###..###...##..#..#.####.###..#....###. -#..#.#..#.#..#.#.#..#....#..#.#....#..# -#..#.#..#.#....##...###..###..#....#..# -###..###..#....#.#..#....#..#.#....###. -#.#..#....#..#.#.#..#....#..#.#....#.#. -#..#.#.....##..#..#.#....###..####.#..# -``` -""" - actual = process_problem_statement(template, "javascript") - expected = """ -```console?lang=javascript&prompt=> -> dots("paper.txt") -###..###...##..#..#.####.###..#....###. -#..#.#..#.#..#.#.#..#....#..#.#....#..# -#..#.#..#.#....##...###..###..#....#..# -###..###..#....#.#..#....#..#.#....###. -#.#..#....#..#.#.#..#....#..#.#....#.#. -#..#.#.....##..#..#.#....###..####.#..# -``` -""" - assert actual == expected - - def test_long_description(): test_dir = Path(__file__).parent description_template = test_dir / "descriptions" / "recoupling.md.j2"