Add support for files to DSL

dodona-edu · Aug 13, 2024 · 7ec058f · 7ec058f
1 parent d54376c
commit 7ec058f
Show file tree

Hide file tree

Showing 3 changed files with 223 additions and 0 deletions.
diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json
@@ -344,6 +344,10 @@
  "description" : "Expected output at stdout",
  "$ref" : "#/definitions/textOutputChannel"
  },
+ "file": {
+ "description" : "Expected files generated by the submission.",
+ "$ref" : "#/definitions/fileOutputChannel"
+ },
  "exit_code" : {
  "type" : "integer",
  "description" : "Expected exit code for the run"
@@ -525,10 +529,101 @@
  "file",
  "data"
  ],
+ "properties" : {
+ "data" : {
+ "$ref" : "#/definitions/textualType"
+ },
+ "oracle" : {
+ "const" : "custom_check"
+ },
+ "file" : {
+ "type" : "string",
+ "description" : "The path to the file containing the custom check function."
+ },
+ "name" : {
+ "type" : "string",
+ "description" : "The name of the custom check function.",
+ "default" : "evaluate"
+ },
+ "arguments" : {
+ "type" : "array",
+ "description" : "List of YAML (or tagged expression) values to use as arguments to the function.",
+ "items" : {
+ "$ref" : "#/definitions/yamlValueOrPythonExpression"
+ }
+ },
+ "languages": {
+ "type" : "array",
+ "description" : "Which programming languages are supported by this oracle.",
+ "items" : {
+ "$ref" : "#/definitions/programmingLanguage"
+ }
+ }
+ }
+ }
+ ]
+ },
+ "fileOutputChannel": {
+ "anyOf" : [
+ {
+ "type" : "object",
+ "description" : "Built-in oracle for files.",
+ "required" : [
+ "content",
+ "location"
+ ],
+ "properties" : {
+ "content" : {
+ "type" : "string",
+ "description" : "Path to the file containing the expected contents, relative to the evaluation directory."
+ },
+ "location" : {
+ "type" : "string",
+ "description" : "Path to where the file generated by the submission should go."
+ },
+ "oracle" : {
+ "const" : "builtin"
+ },
+ "config" : {
+ "anyOf" : [
+ {
+ "$ref" : "#/definitions/textConfigurationOptions"
+ },
+ {
+ "type" : "object",
+ "properties" : {
+ "mode": {
+ "type" : "string",
+ "enum" : ["full", "line"],
+ "default" : "full"
+ }
+ }
+ }
+ ]
+ }
+ }
+ },
+ {
+ "type" : "object",
+ "description" : "Custom oracle for file values.",
+ "required" : [
+ "oracle",
+ "content",
+ "location",
+ "file"
+ ],
  "properties" : {
  "oracle" : {
  "const" : "custom_check"
  },
+ "content" : {
+ "type" : "string",
+ "description" : "Path to the file containing the expected contents, relative to the evaluation directory."
+ },
+ "location" : {
+ "type" : "string",
+ "description" : "Path to where the file generated by the submission should go."
+ },
  "file" : {
  "type" : "string",
  "description" : "The path to the file containing the custom check function."

diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json
@@ -436,6 +436,10 @@
  "description" : "Expected output at stdout",
  "$ref" : "#/definitions/textOutputChannel"
  },
+ "file": {
+ "description" : "Expected files generated by the submission.",
+ "$ref" : "#/definitions/fileOutputChannel"
+ },
  "exit_code" : {
  "type" : "integer",
  "description" : "Expected exit code for the run"
@@ -525,10 +529,101 @@
  "file",
  "data"
  ],
+ "properties" : {
+ "data" : {
+ "$ref" : "#/definitions/textualType"
+ },
+ "oracle" : {
+ "const" : "custom_check"
+ },
+ "file" : {
+ "type" : "string",
+ "description" : "The path to the file containing the custom check function."
+ },
+ "name" : {
+ "type" : "string",
+ "description" : "The name of the custom check function.",
+ "default" : "evaluate"
+ },
+ "arguments" : {
+ "type" : "array",
+ "description" : "List of YAML (or tagged expression) values to use as arguments to the function.",
+ "items" : {
+ "$ref" : "#/definitions/yamlValueOrPythonExpression"
+ }
+ },
+ "languages": {
+ "type" : "array",
+ "description" : "Which programming languages are supported by this oracle.",
+ "items" : {
+ "$ref" : "#/definitions/programmingLanguage"
+ }
+ }
+ }
+ }
+ ]
+ },
+ "fileOutputChannel": {
+ "anyOf" : [
+ {
+ "type" : "object",
+ "description" : "Built-in oracle for files.",
+ "required" : [
+ "content",
+ "location"
+ ],
+ "properties" : {
+ "content" : {
+ "type" : "string",
+ "description" : "Path to the file containing the expected contents, relative to the evaluation directory."
+ },
+ "location" : {
+ "type" : "string",
+ "description" : "Path to where the file generated by the submission should go."
+ },
+ "oracle" : {
+ "const" : "builtin"
+ },
+ "config" : {
+ "anyOf" : [
+ {
+ "$ref" : "#/definitions/textConfigurationOptions"
+ },
+ {
+ "type" : "object",
+ "properties" : {
+ "mode": {
+ "type" : "string",
+ "enum" : ["full", "line"],
+ "default" : "full"
+ }
+ }
+ }
+ ]
+ }
+ }
+ },
+ {
+ "type" : "object",
+ "description" : "Custom oracle for file values.",
+ "required" : [
+ "oracle",
+ "content",
+ "location",
+ "file"
+ ],
  "properties" : {
  "oracle" : {
  "const" : "custom_check"
  },
+ "content" : {
+ "type" : "string",
+ "description" : "Path to the file containing the expected contents, relative to the evaluation directory."
+ },
+ "location" : {
+ "type" : "string",
+ "description" : "Path to where the file generated by the submission should go."
+ },
  "file" : {
  "type" : "string",
  "description" : "The path to the file containing the custom check function."

diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
@@ -52,6 +52,7 @@
  ExceptionOutputChannel,
  ExitCodeOutputChannel,
  ExpectedException,
+ FileOutputChannel,
  FileUrl,
  GenericTextOracle,
  IgnoredChannel,
@@ -63,6 +64,7 @@
  SupportedLanguage,
  Tab,
  Testcase,
+ TextBuiltin,
  TextData,
  TextOutputChannel,
  ValueOutputChannel,
@@ -438,6 +440,35 @@ def _convert_text_output_channel(stream: YamlObject) -> TextOutputChannel:
  raise TypeError(f"Unknown text oracle type: {stream['oracle']}")
 
 
+def _convert_file_output_channel(stream: YamlObject) -> FileOutputChannel:
+ assert isinstance(stream, dict)
+
+ expected = str(stream["content"])
+ actual = str(stream["location"])
+
+ if "oracle" not in stream or stream["oracle"] == "builtin":
+ config = cast(dict, stream.get("config", {}))
+ if "mode" not in config:
+ config["mode"] = "full"
+
+ assert config["mode"] in (
+ "full",
+ "line",
+ ), f"The file oracle only supports modes full and line, not {config['mode']}"
+ return FileOutputChannel(
+ expected_path=expected,
+ actual_path=actual,
+ oracle=GenericTextOracle(name=TextBuiltin.FILE, options=config),
+ )
+ elif stream["oracle"] == "custom_check":
+ return FileOutputChannel(
+ expected_path=expected,
+ actual_path=actual,
+ oracle=_convert_custom_check_oracle(stream),
+ )
+ raise TypeError(f"Unknown file oracle type: {stream['oracle']}")
+
+
 def _convert_yaml_value(stream: YamlObject) -> Value | None:
  if isinstance(stream, ExpressionString):
  # We have an expression string.
@@ -536,6 +567,8 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase:
 
  if (stdout := testcase.get("stdout")) is not None:
  output.stdout = _convert_text_output_channel(stdout)
+ if (file := testcase.get("file")) is not None:
+ output.file = _convert_file_output_channel(file)
  if (stderr := testcase.get("stderr")) is not None:
  output.stderr = _convert_text_output_channel(stderr)
  if (exception := testcase.get("exception")) is not None: