From 7ec058fb90f20460001bcfb622d03fdf33b1a059 Mon Sep 17 00:00:00 2001 From: Niko Strijbol Date: Tue, 13 Aug 2024 18:17:28 +0200 Subject: [PATCH] Add support for files to DSL --- tested/dsl/schema-strict.json | 95 ++++++++++++++++++++++++++++++++++ tested/dsl/schema.json | 95 ++++++++++++++++++++++++++++++++++ tested/dsl/translate_parser.py | 33 ++++++++++++ 3 files changed, 223 insertions(+) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index dac0c4f8..397f2110 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -344,6 +344,10 @@ "description" : "Expected output at stdout", "$ref" : "#/definitions/textOutputChannel" }, + "file": { + "description" : "Expected files generated by the submission.", + "$ref" : "#/definitions/fileOutputChannel" + }, "exit_code" : { "type" : "integer", "description" : "Expected exit code for the run" @@ -525,10 +529,101 @@ "file", "data" ], + "properties" : { + "data" : { + "$ref" : "#/definitions/textualType" + }, + "oracle" : { + "const" : "custom_check" + }, + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + }, + "arguments" : { + "type" : "array", + "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", + "items" : { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type" : "array", + "description" : "Which programming languages are supported by this oracle.", + "items" : { + "$ref" : "#/definitions/programmingLanguage" + } + } + } + } + ] + }, + "fileOutputChannel": { + "anyOf" : [ + { + "type" : "object", + "description" : "Built-in oracle for files.", + "required" : [ + "content", + "location" + ], + "properties" : { + "content" : { + "type" : "string", + "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "location" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + }, + "oracle" : { + "const" : "builtin" + }, + "config" : { + "anyOf" : [ + { + "$ref" : "#/definitions/textConfigurationOptions" + }, + { + "type" : "object", + "properties" : { + "mode": { + "type" : "string", + "enum" : ["full", "line"], + "default" : "full" + } + } + } + ] + } + } + }, + { + "type" : "object", + "description" : "Custom oracle for file values.", + "required" : [ + "oracle", + "content", + "location", + "file" + ], "properties" : { "oracle" : { "const" : "custom_check" }, + "content" : { + "type" : "string", + "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "location" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + }, "file" : { "type" : "string", "description" : "The path to the file containing the custom check function." diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index ca56d53d..e9896ff6 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -436,6 +436,10 @@ "description" : "Expected output at stdout", "$ref" : "#/definitions/textOutputChannel" }, + "file": { + "description" : "Expected files generated by the submission.", + "$ref" : "#/definitions/fileOutputChannel" + }, "exit_code" : { "type" : "integer", "description" : "Expected exit code for the run" @@ -525,10 +529,101 @@ "file", "data" ], + "properties" : { + "data" : { + "$ref" : "#/definitions/textualType" + }, + "oracle" : { + "const" : "custom_check" + }, + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + }, + "arguments" : { + "type" : "array", + "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", + "items" : { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type" : "array", + "description" : "Which programming languages are supported by this oracle.", + "items" : { + "$ref" : "#/definitions/programmingLanguage" + } + } + } + } + ] + }, + "fileOutputChannel": { + "anyOf" : [ + { + "type" : "object", + "description" : "Built-in oracle for files.", + "required" : [ + "content", + "location" + ], + "properties" : { + "content" : { + "type" : "string", + "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "location" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + }, + "oracle" : { + "const" : "builtin" + }, + "config" : { + "anyOf" : [ + { + "$ref" : "#/definitions/textConfigurationOptions" + }, + { + "type" : "object", + "properties" : { + "mode": { + "type" : "string", + "enum" : ["full", "line"], + "default" : "full" + } + } + } + ] + } + } + }, + { + "type" : "object", + "description" : "Custom oracle for file values.", + "required" : [ + "oracle", + "content", + "location", + "file" + ], "properties" : { "oracle" : { "const" : "custom_check" }, + "content" : { + "type" : "string", + "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "location" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + }, "file" : { "type" : "string", "description" : "The path to the file containing the custom check function." diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index fc119697..fa7d4e02 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -52,6 +52,7 @@ ExceptionOutputChannel, ExitCodeOutputChannel, ExpectedException, + FileOutputChannel, FileUrl, GenericTextOracle, IgnoredChannel, @@ -63,6 +64,7 @@ SupportedLanguage, Tab, Testcase, + TextBuiltin, TextData, TextOutputChannel, ValueOutputChannel, @@ -438,6 +440,35 @@ def _convert_text_output_channel(stream: YamlObject) -> TextOutputChannel: raise TypeError(f"Unknown text oracle type: {stream['oracle']}") +def _convert_file_output_channel(stream: YamlObject) -> FileOutputChannel: + assert isinstance(stream, dict) + + expected = str(stream["content"]) + actual = str(stream["location"]) + + if "oracle" not in stream or stream["oracle"] == "builtin": + config = cast(dict, stream.get("config", {})) + if "mode" not in config: + config["mode"] = "full" + + assert config["mode"] in ( + "full", + "line", + ), f"The file oracle only supports modes full and line, not {config['mode']}" + return FileOutputChannel( + expected_path=expected, + actual_path=actual, + oracle=GenericTextOracle(name=TextBuiltin.FILE, options=config), + ) + elif stream["oracle"] == "custom_check": + return FileOutputChannel( + expected_path=expected, + actual_path=actual, + oracle=_convert_custom_check_oracle(stream), + ) + raise TypeError(f"Unknown file oracle type: {stream['oracle']}") + + def _convert_yaml_value(stream: YamlObject) -> Value | None: if isinstance(stream, ExpressionString): # We have an expression string. @@ -536,6 +567,8 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: if (stdout := testcase.get("stdout")) is not None: output.stdout = _convert_text_output_channel(stdout) + if (file := testcase.get("file")) is not None: + output.file = _convert_file_output_channel(file) if (stderr := testcase.get("stderr")) is not None: output.stderr = _convert_text_output_channel(stderr) if (exception := testcase.get("exception")) is not None: