From d54376c3f07bd03fcd00b1457c50426517e58af4 Mon Sep 17 00:00:00 2001 From: Niko Strijbol Date: Tue, 13 Aug 2024 18:17:12 +0200 Subject: [PATCH 1/4] Fix Monad issue in Haskell generator --- tested/languages/haskell/generators.py | 32 +++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tested/languages/haskell/generators.py b/tested/languages/haskell/generators.py index ffcc68f5..c2897db7 100644 --- a/tested/languages/haskell/generators.py +++ b/tested/languages/haskell/generators.py @@ -252,7 +252,7 @@ def convert_execution_unit(pu: PreparedExecutionUnit) -> str: ) else: assert isinstance(tc.input, PreparedTestcaseStatement) - # In Haskell we do not actually have statements, so we need to keep them separate. + # In Haskell we do not have statements, so we need to keep them separate. # Additionally, exceptions with "statements" are not supported at this time. if is_statement_strict(tc.input.statement): result += ( @@ -262,30 +262,30 @@ def convert_execution_unit(pu: PreparedExecutionUnit) -> str: ) else: result += indent + f"result{i1} <- catch\n" - result += ( - indent * 2 - + f"({convert_statement(tc.input.unwrapped_input_statement(), True)}\n" - ) + result += indent * 2 + f"(do\n" # Start a `do` block id_result = tc.input.input_statement("r") + result += ( + indent * 3 + + f"r <- {convert_statement(tc.input.unwrapped_input_statement(), True)}\n" + ) # Bind the result to 'r' if isinstance(id_result, Identifier): - # In this case, we don't catch the value in the sendValues function. - # This results in pure value instead of an IO monad, so lift the value into the monad. result += ( - indent * 3 - + f">>= \\r -> return {convert_statement(id_result)}\n" - ) + indent * 3 + f"sendValue {convert_statement(id_result)}\n" + ) # Send the value if it's an identifier else: result += ( - indent * 3 + f">>= \\r -> {convert_statement(id_result)}\n" - ) + indent * 3 + convert_statement(id_result) + "\n" + ) # Otherwise, execute the statement + result += indent * 3 + "return ()\n" # Explicitly return unit + result += indent * 2 + ") (\\e -> do\n" # Handle exceptions result += ( indent * 3 - + f">> let ee = (Nothing :: Maybe SomeException) in {convert_statement(tc.exception_statement('ee'))})\n" + + f"let ee = (Just (e :: SomeException)) in {convert_statement(tc.exception_statement('ee'))}\n" ) result += ( - indent * 2 - + f"(\\e -> let ee = (Just (e :: SomeException)) in {convert_statement(tc.exception_statement('ee'))})\n" - ) + indent * 3 + "return ()\n" + ) # Explicitly return unit in the exception case + result += indent * 2 + ")\n" # End the `catch` block result += indent + ctx.after + "\n" result += indent + 'putStr ""\n' From 7ec058fb90f20460001bcfb622d03fdf33b1a059 Mon Sep 17 00:00:00 2001 From: Niko Strijbol Date: Tue, 13 Aug 2024 18:17:28 +0200 Subject: [PATCH 2/4] Add support for files to DSL --- tested/dsl/schema-strict.json | 95 ++++++++++++++++++++++++++++++++++ tested/dsl/schema.json | 95 ++++++++++++++++++++++++++++++++++ tested/dsl/translate_parser.py | 33 ++++++++++++ 3 files changed, 223 insertions(+) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index dac0c4f8..397f2110 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -344,6 +344,10 @@ "description" : "Expected output at stdout", "$ref" : "#/definitions/textOutputChannel" }, + "file": { + "description" : "Expected files generated by the submission.", + "$ref" : "#/definitions/fileOutputChannel" + }, "exit_code" : { "type" : "integer", "description" : "Expected exit code for the run" @@ -525,10 +529,101 @@ "file", "data" ], + "properties" : { + "data" : { + "$ref" : "#/definitions/textualType" + }, + "oracle" : { + "const" : "custom_check" + }, + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + }, + "arguments" : { + "type" : "array", + "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", + "items" : { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type" : "array", + "description" : "Which programming languages are supported by this oracle.", + "items" : { + "$ref" : "#/definitions/programmingLanguage" + } + } + } + } + ] + }, + "fileOutputChannel": { + "anyOf" : [ + { + "type" : "object", + "description" : "Built-in oracle for files.", + "required" : [ + "content", + "location" + ], + "properties" : { + "content" : { + "type" : "string", + "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "location" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + }, + "oracle" : { + "const" : "builtin" + }, + "config" : { + "anyOf" : [ + { + "$ref" : "#/definitions/textConfigurationOptions" + }, + { + "type" : "object", + "properties" : { + "mode": { + "type" : "string", + "enum" : ["full", "line"], + "default" : "full" + } + } + } + ] + } + } + }, + { + "type" : "object", + "description" : "Custom oracle for file values.", + "required" : [ + "oracle", + "content", + "location", + "file" + ], "properties" : { "oracle" : { "const" : "custom_check" }, + "content" : { + "type" : "string", + "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "location" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + }, "file" : { "type" : "string", "description" : "The path to the file containing the custom check function." diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index ca56d53d..e9896ff6 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -436,6 +436,10 @@ "description" : "Expected output at stdout", "$ref" : "#/definitions/textOutputChannel" }, + "file": { + "description" : "Expected files generated by the submission.", + "$ref" : "#/definitions/fileOutputChannel" + }, "exit_code" : { "type" : "integer", "description" : "Expected exit code for the run" @@ -525,10 +529,101 @@ "file", "data" ], + "properties" : { + "data" : { + "$ref" : "#/definitions/textualType" + }, + "oracle" : { + "const" : "custom_check" + }, + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + }, + "arguments" : { + "type" : "array", + "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", + "items" : { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type" : "array", + "description" : "Which programming languages are supported by this oracle.", + "items" : { + "$ref" : "#/definitions/programmingLanguage" + } + } + } + } + ] + }, + "fileOutputChannel": { + "anyOf" : [ + { + "type" : "object", + "description" : "Built-in oracle for files.", + "required" : [ + "content", + "location" + ], + "properties" : { + "content" : { + "type" : "string", + "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "location" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + }, + "oracle" : { + "const" : "builtin" + }, + "config" : { + "anyOf" : [ + { + "$ref" : "#/definitions/textConfigurationOptions" + }, + { + "type" : "object", + "properties" : { + "mode": { + "type" : "string", + "enum" : ["full", "line"], + "default" : "full" + } + } + } + ] + } + } + }, + { + "type" : "object", + "description" : "Custom oracle for file values.", + "required" : [ + "oracle", + "content", + "location", + "file" + ], "properties" : { "oracle" : { "const" : "custom_check" }, + "content" : { + "type" : "string", + "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "location" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + }, "file" : { "type" : "string", "description" : "The path to the file containing the custom check function." diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index fc119697..fa7d4e02 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -52,6 +52,7 @@ ExceptionOutputChannel, ExitCodeOutputChannel, ExpectedException, + FileOutputChannel, FileUrl, GenericTextOracle, IgnoredChannel, @@ -63,6 +64,7 @@ SupportedLanguage, Tab, Testcase, + TextBuiltin, TextData, TextOutputChannel, ValueOutputChannel, @@ -438,6 +440,35 @@ def _convert_text_output_channel(stream: YamlObject) -> TextOutputChannel: raise TypeError(f"Unknown text oracle type: {stream['oracle']}") +def _convert_file_output_channel(stream: YamlObject) -> FileOutputChannel: + assert isinstance(stream, dict) + + expected = str(stream["content"]) + actual = str(stream["location"]) + + if "oracle" not in stream or stream["oracle"] == "builtin": + config = cast(dict, stream.get("config", {})) + if "mode" not in config: + config["mode"] = "full" + + assert config["mode"] in ( + "full", + "line", + ), f"The file oracle only supports modes full and line, not {config['mode']}" + return FileOutputChannel( + expected_path=expected, + actual_path=actual, + oracle=GenericTextOracle(name=TextBuiltin.FILE, options=config), + ) + elif stream["oracle"] == "custom_check": + return FileOutputChannel( + expected_path=expected, + actual_path=actual, + oracle=_convert_custom_check_oracle(stream), + ) + raise TypeError(f"Unknown file oracle type: {stream['oracle']}") + + def _convert_yaml_value(stream: YamlObject) -> Value | None: if isinstance(stream, ExpressionString): # We have an expression string. @@ -536,6 +567,8 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: if (stdout := testcase.get("stdout")) is not None: output.stdout = _convert_text_output_channel(stdout) + if (file := testcase.get("file")) is not None: + output.file = _convert_file_output_channel(file) if (stderr := testcase.get("stderr")) is not None: output.stderr = _convert_text_output_channel(stderr) if (exception := testcase.get("exception")) is not None: From 43a00e595c37c867554c7d33c4ceb41d1fb252db Mon Sep 17 00:00:00 2001 From: Niko Strijbol Date: Tue, 13 Aug 2024 18:18:37 +0200 Subject: [PATCH 3/4] Add tests for files in DSL --- .../evaluation/one.tson | 0 .../solution/correct-async.js | 0 .../solution/correct.c | 0 .../solution/correct.cs | 0 .../solution/correct.hs | 0 .../solution/correct.java | 0 .../solution/correct.js | 0 .../solution/correct.kt | 0 .../solution/correct.py | 0 .../solution/correct.sh | 0 .../workdir/data.txt | 0 .../evaluation/contents.txt | 1 + .../evaluation/one.yaml | 7 +++++ .../solution/correct.c | 7 +++++ .../solution/correct.cs | 12 +++++++++ .../solution/correct.hs | 7 +++++ .../solution/correct.java | 15 +++++++++++ .../solution/correct.js | 5 ++++ .../solution/correct.kt | 5 ++++ .../solution/correct.py | 3 +++ .../solution/correct.sh | 4 +++ .../evaluation/javascript-object.yaml | 1 - tests/test_io_exercises.py | 26 +++++++++++++++++-- tests/test_language_quircks.py | 2 +- 24 files changed, 91 insertions(+), 4 deletions(-) rename tests/exercises/{echo-function-file => echo-function-file-input}/evaluation/one.tson (100%) rename tests/exercises/{echo-function-file => echo-function-file-input}/solution/correct-async.js (100%) rename tests/exercises/{echo-function-file => echo-function-file-input}/solution/correct.c (100%) rename tests/exercises/{echo-function-file => echo-function-file-input}/solution/correct.cs (100%) rename tests/exercises/{echo-function-file => echo-function-file-input}/solution/correct.hs (100%) rename tests/exercises/{echo-function-file => echo-function-file-input}/solution/correct.java (100%) rename tests/exercises/{echo-function-file => echo-function-file-input}/solution/correct.js (100%) rename tests/exercises/{echo-function-file => echo-function-file-input}/solution/correct.kt (100%) rename tests/exercises/{echo-function-file => echo-function-file-input}/solution/correct.py (100%) rename tests/exercises/{echo-function-file => echo-function-file-input}/solution/correct.sh (100%) rename tests/exercises/{echo-function-file => echo-function-file-input}/workdir/data.txt (100%) create mode 100644 tests/exercises/echo-function-file-output/evaluation/contents.txt create mode 100644 tests/exercises/echo-function-file-output/evaluation/one.yaml create mode 100644 tests/exercises/echo-function-file-output/solution/correct.c create mode 100644 tests/exercises/echo-function-file-output/solution/correct.cs create mode 100644 tests/exercises/echo-function-file-output/solution/correct.hs create mode 100644 tests/exercises/echo-function-file-output/solution/correct.java create mode 100644 tests/exercises/echo-function-file-output/solution/correct.js create mode 100644 tests/exercises/echo-function-file-output/solution/correct.kt create mode 100644 tests/exercises/echo-function-file-output/solution/correct.py create mode 100644 tests/exercises/echo-function-file-output/solution/correct.sh diff --git a/tests/exercises/echo-function-file/evaluation/one.tson b/tests/exercises/echo-function-file-input/evaluation/one.tson similarity index 100% rename from tests/exercises/echo-function-file/evaluation/one.tson rename to tests/exercises/echo-function-file-input/evaluation/one.tson diff --git a/tests/exercises/echo-function-file/solution/correct-async.js b/tests/exercises/echo-function-file-input/solution/correct-async.js similarity index 100% rename from tests/exercises/echo-function-file/solution/correct-async.js rename to tests/exercises/echo-function-file-input/solution/correct-async.js diff --git a/tests/exercises/echo-function-file/solution/correct.c b/tests/exercises/echo-function-file-input/solution/correct.c similarity index 100% rename from tests/exercises/echo-function-file/solution/correct.c rename to tests/exercises/echo-function-file-input/solution/correct.c diff --git a/tests/exercises/echo-function-file/solution/correct.cs b/tests/exercises/echo-function-file-input/solution/correct.cs similarity index 100% rename from tests/exercises/echo-function-file/solution/correct.cs rename to tests/exercises/echo-function-file-input/solution/correct.cs diff --git a/tests/exercises/echo-function-file/solution/correct.hs b/tests/exercises/echo-function-file-input/solution/correct.hs similarity index 100% rename from tests/exercises/echo-function-file/solution/correct.hs rename to tests/exercises/echo-function-file-input/solution/correct.hs diff --git a/tests/exercises/echo-function-file/solution/correct.java b/tests/exercises/echo-function-file-input/solution/correct.java similarity index 100% rename from tests/exercises/echo-function-file/solution/correct.java rename to tests/exercises/echo-function-file-input/solution/correct.java diff --git a/tests/exercises/echo-function-file/solution/correct.js b/tests/exercises/echo-function-file-input/solution/correct.js similarity index 100% rename from tests/exercises/echo-function-file/solution/correct.js rename to tests/exercises/echo-function-file-input/solution/correct.js diff --git a/tests/exercises/echo-function-file/solution/correct.kt b/tests/exercises/echo-function-file-input/solution/correct.kt similarity index 100% rename from tests/exercises/echo-function-file/solution/correct.kt rename to tests/exercises/echo-function-file-input/solution/correct.kt diff --git a/tests/exercises/echo-function-file/solution/correct.py b/tests/exercises/echo-function-file-input/solution/correct.py similarity index 100% rename from tests/exercises/echo-function-file/solution/correct.py rename to tests/exercises/echo-function-file-input/solution/correct.py diff --git a/tests/exercises/echo-function-file/solution/correct.sh b/tests/exercises/echo-function-file-input/solution/correct.sh similarity index 100% rename from tests/exercises/echo-function-file/solution/correct.sh rename to tests/exercises/echo-function-file-input/solution/correct.sh diff --git a/tests/exercises/echo-function-file/workdir/data.txt b/tests/exercises/echo-function-file-input/workdir/data.txt similarity index 100% rename from tests/exercises/echo-function-file/workdir/data.txt rename to tests/exercises/echo-function-file-input/workdir/data.txt diff --git a/tests/exercises/echo-function-file-output/evaluation/contents.txt b/tests/exercises/echo-function-file-output/evaluation/contents.txt new file mode 100644 index 00000000..b5fc21b3 --- /dev/null +++ b/tests/exercises/echo-function-file-output/evaluation/contents.txt @@ -0,0 +1 @@ +Hallo diff --git a/tests/exercises/echo-function-file-output/evaluation/one.yaml b/tests/exercises/echo-function-file-output/evaluation/one.yaml new file mode 100644 index 00000000..a428bf27 --- /dev/null +++ b/tests/exercises/echo-function-file-output/evaluation/one.yaml @@ -0,0 +1,7 @@ +- tab: "Test" + testcases: + - statement: echo_function("result.txt", "Hallo") + file: + content: "contents.txt" + location: "result.txt" + oracle: builtin diff --git a/tests/exercises/echo-function-file-output/solution/correct.c b/tests/exercises/echo-function-file-output/solution/correct.c new file mode 100644 index 00000000..3e95055e --- /dev/null +++ b/tests/exercises/echo-function-file-output/solution/correct.c @@ -0,0 +1,7 @@ +#include + +void echo_function(const char *filename, const char *string) { + FILE *file = fopen(filename, "w"); + fprintf(file, "%s\n", string); + fclose(file); +} diff --git a/tests/exercises/echo-function-file-output/solution/correct.cs b/tests/exercises/echo-function-file-output/solution/correct.cs new file mode 100644 index 00000000..fa06764f --- /dev/null +++ b/tests/exercises/echo-function-file-output/solution/correct.cs @@ -0,0 +1,12 @@ +using System.IO; + +class Submission +{ + public static void EchoFunction(string filename, string stringToWrite) + { + using (StreamWriter writer = new StreamWriter(filename)) + { + writer.WriteLine(stringToWrite); + } + } +} diff --git a/tests/exercises/echo-function-file-output/solution/correct.hs b/tests/exercises/echo-function-file-output/solution/correct.hs new file mode 100644 index 00000000..b7a2d074 --- /dev/null +++ b/tests/exercises/echo-function-file-output/solution/correct.hs @@ -0,0 +1,7 @@ +import System.IO (writeFile) + +echoFunction :: FilePath -> String -> IO () +echoFunction filename content = do + let contentWithNewline = content ++ "\n" + writeFile filename contentWithNewline + return () diff --git a/tests/exercises/echo-function-file-output/solution/correct.java b/tests/exercises/echo-function-file-output/solution/correct.java new file mode 100644 index 00000000..5e1138ef --- /dev/null +++ b/tests/exercises/echo-function-file-output/solution/correct.java @@ -0,0 +1,15 @@ +import java.io.FileWriter; +import java.io.IOException; + +public class Submission { + + public static void echoFunction(String filename, String stringToWrite) { + try (FileWriter writer = new FileWriter(filename)) { + writer.write(stringToWrite); + writer.write(System.lineSeparator()); // Add a newline + } catch (IOException e) { + System.err.println("Error writing to file: " + e.getMessage()); + } + } +} + diff --git a/tests/exercises/echo-function-file-output/solution/correct.js b/tests/exercises/echo-function-file-output/solution/correct.js new file mode 100644 index 00000000..da5cfb3d --- /dev/null +++ b/tests/exercises/echo-function-file-output/solution/correct.js @@ -0,0 +1,5 @@ +const fs = require('fs'); + +function echoFunction(filename, stringToWrite) { + fs.writeFileSync(filename, stringToWrite + '\n', { flag: 'w' }); +} diff --git a/tests/exercises/echo-function-file-output/solution/correct.kt b/tests/exercises/echo-function-file-output/solution/correct.kt new file mode 100644 index 00000000..df3d38cd --- /dev/null +++ b/tests/exercises/echo-function-file-output/solution/correct.kt @@ -0,0 +1,5 @@ +import java.io.File + +fun echoFunction(filename: String, stringToWrite: String) { + File(filename).writeText(stringToWrite + "\n") +} diff --git a/tests/exercises/echo-function-file-output/solution/correct.py b/tests/exercises/echo-function-file-output/solution/correct.py new file mode 100644 index 00000000..28703702 --- /dev/null +++ b/tests/exercises/echo-function-file-output/solution/correct.py @@ -0,0 +1,3 @@ +def echo_function(filename, string_to_write): + with open(filename, 'w') as file: + file.write(string_to_write + '\n') # Write the string and a newline) diff --git a/tests/exercises/echo-function-file-output/solution/correct.sh b/tests/exercises/echo-function-file-output/solution/correct.sh new file mode 100644 index 00000000..8fabf9af --- /dev/null +++ b/tests/exercises/echo-function-file-output/solution/correct.sh @@ -0,0 +1,4 @@ +echo_function() { + echo "$2" > "$1" +} + diff --git a/tests/exercises/echo-function/evaluation/javascript-object.yaml b/tests/exercises/echo-function/evaluation/javascript-object.yaml index a6ae3414..fb6eb476 100644 --- a/tests/exercises/echo-function/evaluation/javascript-object.yaml +++ b/tests/exercises/echo-function/evaluation/javascript-object.yaml @@ -3,4 +3,3 @@ - testcases: - expression: 'echo("input-1")' return: !object {} - diff --git a/tests/test_io_exercises.py b/tests/test_io_exercises.py index fc1e971c..99bd709e 100644 --- a/tests/test_io_exercises.py +++ b/tests/test_io_exercises.py @@ -44,11 +44,16 @@ def test_io_function_exercise( @pytest.mark.parametrize("language", ALL_LANGUAGES) -def test_io_function_file_exercise( +def test_io_function_file_input_exercise( language: str, tmp_path: Path, pytestconfig: pytest.Config ): conf = configuration( - pytestconfig, "echo-function-file", language, tmp_path, "one.tson", "correct" + pytestconfig, + "echo-function-file-input", + language, + tmp_path, + "one.tson", + "correct", ) shutil.copytree( Path(conf.resources).parent / "workdir", tmp_path, dirs_exist_ok=True @@ -58,6 +63,23 @@ def test_io_function_file_exercise( assert updates.find_status_enum() == ["correct"] +@pytest.mark.parametrize("language", ALL_LANGUAGES) +def test_io_function_file_output_exercise( + language: str, tmp_path: Path, pytestconfig: pytest.Config +): + conf = configuration( + pytestconfig, + "echo-function-file-output", + language, + tmp_path, + "one.yaml", + "correct", + ) + result = execute_config(conf) + updates = assert_valid_output(result, pytestconfig) + assert updates.find_status_enum() == ["correct"] + + @pytest.mark.parametrize("language", ALL_LANGUAGES) def test_io_function_additional_source_files( language: str, tmp_path: Path, pytestconfig: pytest.Config diff --git a/tests/test_language_quircks.py b/tests/test_language_quircks.py index dbb0c9bc..9af65f88 100644 --- a/tests/test_language_quircks.py +++ b/tests/test_language_quircks.py @@ -152,7 +152,7 @@ def test_javascript_exception_missing_message( assert updates.find_status_enum() == ["wrong"] -@pytest.mark.parametrize("exercise", ["echo-function-file", "echo-function"]) +@pytest.mark.parametrize("exercise", ["echo-function-file-input", "echo-function"]) def test_javascript_async(exercise: str, tmp_path: Path, pytestconfig: pytest.Config): conf = configuration( pytestconfig, exercise, "javascript", tmp_path, "one.tson", "correct-async" From 28776e5ffeaf2b357bf9857abce2c93e12ea3365 Mon Sep 17 00:00:00 2001 From: Niko Strijbol Date: Wed, 14 Aug 2024 12:02:18 +0200 Subject: [PATCH 4/4] Add another test --- tests/test_dsl_yaml.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index 497f1da4..0f45d701 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -31,6 +31,7 @@ ) from tested.testsuite import ( CustomCheckOracle, + FileOutputChannel, FileUrl, GenericTextOracle, GenericValueOracle, @@ -657,6 +658,37 @@ def test_value_built_in_checks_implied(): ) +def test_file_custom_check_correct(): + yaml_str = f""" + - tab: 'Test' + contexts: + - testcases: + - statement: 'test()' + file: + content: "test/hallo.txt" + oracle: "custom_check" + location: "test.txt" + name: "evaluate_test" + file: "test.py" + """ + json_str = translate_to_test_suite(yaml_str) + suite = parse_test_suite(json_str) + assert len(suite.tabs) == 1 + tab = suite.tabs[0] + assert len(tab.contexts) == 1 + testcases = tab.contexts[0].testcases + assert len(testcases) == 1 + test = testcases[0] + assert isinstance(test.input, FunctionCall) + assert isinstance(test.output.file, FileOutputChannel) + assert isinstance(test.output.file.oracle, CustomCheckOracle) + assert test.output.file.actual_path == "test.txt" + assert test.output.file.expected_path == "test/hallo.txt" + oracle = test.output.file.oracle + assert oracle.function.name == "evaluate_test" + assert oracle.function.file == Path("test.py") + + def test_value_built_in_checks_explicit(): yaml_str = f""" - tab: 'Test'