Merge pull request #535 from dodona-edu/feature/files-in-dsl

Add support for file evaluation in DSL

tested/dsl/schema-strict.json

@@ -344,6 +344,10 @@
  "description" : "Expected output at stdout",
  "$ref" : "#/definitions/textOutputChannel"
  },
+ "file": {
+ "description" : "Expected files generated by the submission.",
+ "$ref" : "#/definitions/fileOutputChannel"
+ },
  "exit_code" : {
  "type" : "integer",
  "description" : "Expected exit code for the run"
@@ -525,10 +529,101 @@
  "file",
  "data"
  ],
+ "properties" : {
+ "data" : {
+ "$ref" : "#/definitions/textualType"
+ },
+ "oracle" : {
+ "const" : "custom_check"
+ },
+ "file" : {
+ "type" : "string",
+ "description" : "The path to the file containing the custom check function."
+ },
+ "name" : {
+ "type" : "string",
+ "description" : "The name of the custom check function.",
+ "default" : "evaluate"
+ },
+ "arguments" : {
+ "type" : "array",
+ "description" : "List of YAML (or tagged expression) values to use as arguments to the function.",
+ "items" : {
+ "$ref" : "#/definitions/yamlValueOrPythonExpression"
+ }
+ },
+ "languages": {
+ "type" : "array",
+ "description" : "Which programming languages are supported by this oracle.",
+ "items" : {
+ "$ref" : "#/definitions/programmingLanguage"
+ }
+ }
+ }
+ }
+ ]
+ },
+ "fileOutputChannel": {
+ "anyOf" : [
+ {
+ "type" : "object",
+ "description" : "Built-in oracle for files.",
+ "required" : [
+ "content",
+ "location"
+ ],
+ "properties" : {
+ "content" : {
+ "type" : "string",
+ "description" : "Path to the file containing the expected contents, relative to the evaluation directory."
+ },
+ "location" : {
+ "type" : "string",
+ "description" : "Path to where the file generated by the submission should go."
+ },
+ "oracle" : {
+ "const" : "builtin"
+ },
+ "config" : {
+ "anyOf" : [
+ {
+ "$ref" : "#/definitions/textConfigurationOptions"
+ },
+ {
+ "type" : "object",
+ "properties" : {
+ "mode": {
+ "type" : "string",
+ "enum" : ["full", "line"],
+ "default" : "full"
+ }
+ }
+ }
+ ]
+ }
+ }
+ },
+ {
+ "type" : "object",
+ "description" : "Custom oracle for file values.",
+ "required" : [
+ "oracle",
+ "content",
+ "location",
+ "file"
+ ],
  "properties" : {
  "oracle" : {
  "const" : "custom_check"
  },
+ "content" : {
+ "type" : "string",
+ "description" : "Path to the file containing the expected contents, relative to the evaluation directory."
+ },
+ "location" : {
+ "type" : "string",
+ "description" : "Path to where the file generated by the submission should go."
+ },
  "file" : {
  "type" : "string",
  "description" : "The path to the file containing the custom check function."

tested/dsl/schema.json

@@ -436,6 +436,10 @@
  "description" : "Expected output at stdout",
  "$ref" : "#/definitions/textOutputChannel"
  },
+ "file": {
+ "description" : "Expected files generated by the submission.",
+ "$ref" : "#/definitions/fileOutputChannel"
+ },
  "exit_code" : {
  "type" : "integer",
  "description" : "Expected exit code for the run"
@@ -525,10 +529,101 @@
  "file",
  "data"
  ],
+ "properties" : {
+ "data" : {
+ "$ref" : "#/definitions/textualType"
+ },
+ "oracle" : {
+ "const" : "custom_check"
+ },
+ "file" : {
+ "type" : "string",
+ "description" : "The path to the file containing the custom check function."
+ },
+ "name" : {
+ "type" : "string",
+ "description" : "The name of the custom check function.",
+ "default" : "evaluate"
+ },
+ "arguments" : {
+ "type" : "array",
+ "description" : "List of YAML (or tagged expression) values to use as arguments to the function.",
+ "items" : {
+ "$ref" : "#/definitions/yamlValueOrPythonExpression"
+ }
+ },
+ "languages": {
+ "type" : "array",
+ "description" : "Which programming languages are supported by this oracle.",
+ "items" : {
+ "$ref" : "#/definitions/programmingLanguage"
+ }
+ }
+ }
+ }
+ ]
+ },
+ "fileOutputChannel": {
+ "anyOf" : [
+ {
+ "type" : "object",
+ "description" : "Built-in oracle for files.",
+ "required" : [
+ "content",
+ "location"
+ ],
+ "properties" : {
+ "content" : {
+ "type" : "string",
+ "description" : "Path to the file containing the expected contents, relative to the evaluation directory."
+ },
+ "location" : {
+ "type" : "string",
+ "description" : "Path to where the file generated by the submission should go."
+ },
+ "oracle" : {
+ "const" : "builtin"
+ },
+ "config" : {
+ "anyOf" : [
+ {
+ "$ref" : "#/definitions/textConfigurationOptions"
+ },
+ {
+ "type" : "object",
+ "properties" : {
+ "mode": {
+ "type" : "string",
+ "enum" : ["full", "line"],
+ "default" : "full"
+ }
+ }
+ }
+ ]
+ }
+ }
+ },
+ {
+ "type" : "object",
+ "description" : "Custom oracle for file values.",
+ "required" : [
+ "oracle",
+ "content",
+ "location",
+ "file"
+ ],
  "properties" : {
  "oracle" : {
  "const" : "custom_check"
  },
+ "content" : {
+ "type" : "string",
+ "description" : "Path to the file containing the expected contents, relative to the evaluation directory."
+ },
+ "location" : {
+ "type" : "string",
+ "description" : "Path to where the file generated by the submission should go."
+ },
  "file" : {
  "type" : "string",
  "description" : "The path to the file containing the custom check function."

tested/dsl/translate_parser.py

@@ -52,6 +52,7 @@
  ExceptionOutputChannel,
  ExitCodeOutputChannel,
  ExpectedException,
+ FileOutputChannel,
  FileUrl,
  GenericTextOracle,
  IgnoredChannel,
@@ -63,6 +64,7 @@
  SupportedLanguage,
  Tab,
  Testcase,
+ TextBuiltin,
  TextData,
  TextOutputChannel,
  ValueOutputChannel,
@@ -438,6 +440,35 @@ def _convert_text_output_channel(stream: YamlObject) -> TextOutputChannel:
  raise TypeError(f"Unknown text oracle type: {stream['oracle']}")
 
 
+def _convert_file_output_channel(stream: YamlObject) -> FileOutputChannel:
+ assert isinstance(stream, dict)
+
+ expected = str(stream["content"])
+ actual = str(stream["location"])
+
+ if "oracle" not in stream or stream["oracle"] == "builtin":
+ config = cast(dict, stream.get("config", {}))
+ if "mode" not in config:
+ config["mode"] = "full"
+
+ assert config["mode"] in (
+ "full",
+ "line",
+ ), f"The file oracle only supports modes full and line, not {config['mode']}"
+ return FileOutputChannel(
+ expected_path=expected,
+ actual_path=actual,
+ oracle=GenericTextOracle(name=TextBuiltin.FILE, options=config),
+ )
+ elif stream["oracle"] == "custom_check":
+ return FileOutputChannel(
+ expected_path=expected,
+ actual_path=actual,
+ oracle=_convert_custom_check_oracle(stream),
+ )
+ raise TypeError(f"Unknown file oracle type: {stream['oracle']}")
+
+
 def _convert_yaml_value(stream: YamlObject) -> Value | None:
  if isinstance(stream, ExpressionString):
  # We have an expression string.
@@ -536,6 +567,8 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase:
 
  if (stdout := testcase.get("stdout")) is not None:
  output.stdout = _convert_text_output_channel(stdout)
+ if (file := testcase.get("file")) is not None:
+ output.file = _convert_file_output_channel(file)
  if (stderr := testcase.get("stderr")) is not None:
  output.stderr = _convert_text_output_channel(stderr)
  if (exception := testcase.get("exception")) is not None:

tested/languages/haskell/generators.py

@@ -252,7 +252,7 @@ def convert_execution_unit(pu: PreparedExecutionUnit) -> str:
  )
  else:
  assert isinstance(tc.input, PreparedTestcaseStatement)
- # In Haskell we do not actually have statements, so we need to keep them separate.
+ # In Haskell we do not have statements, so we need to keep them separate.
  # Additionally, exceptions with "statements" are not supported at this time.
  if is_statement_strict(tc.input.statement):
  result += (
@@ -262,30 +262,30 @@ def convert_execution_unit(pu: PreparedExecutionUnit) -> str:
  )
  else:
  result += indent + f"result{i1} <- catch\n"
- result += (
- indent * 2
- + f"({convert_statement(tc.input.unwrapped_input_statement(), True)}\n"
- )
+ result += indent * 2 + f"(do\n" # Start a `do` block
  id_result = tc.input.input_statement("r")
+ result += (
+ indent * 3
+ + f"r <- {convert_statement(tc.input.unwrapped_input_statement(), True)}\n"
+ ) # Bind the result to 'r'
  if isinstance(id_result, Identifier):
- # In this case, we don't catch the value in the sendValues function.
- # This results in pure value instead of an IO monad, so lift the value into the monad.
  result += (
- indent * 3
- + f">>= \\r -> return {convert_statement(id_result)}\n"
- )
+ indent * 3 + f"sendValue {convert_statement(id_result)}\n"
+ ) # Send the value if it's an identifier
  else:
  result += (
- indent * 3 + f">>= \\r -> {convert_statement(id_result)}\n"
- )
+ indent * 3 + convert_statement(id_result) + "\n"
+ ) # Otherwise, execute the statement
+ result += indent * 3 + "return ()\n" # Explicitly return unit
+ result += indent * 2 + ") (\\e -> do\n" # Handle exceptions
  result += (
  indent * 3
- + f">> let ee = (Nothing :: Maybe SomeException) in {convert_statement(tc.exception_statement('ee'))})\n"
+ + f"let ee = (Just (e :: SomeException)) in {convert_statement(tc.exception_statement('ee'))}\n"
  )
  result += (
- indent * 2
-  + f"(\\e -> let ee = (Just (e :: SomeException)) in {convert_statement(tc.exception_statement('ee'))})\n"
- )
+ indent * 3 + "return ()\n"
+ ) # Explicitly return unit in the exception case
+ result += indent * 2 + ")\n" # End the `catch` block
  result += indent + ctx.after + "\n"
  result += indent + 'putStr ""\n'
 

tests/exercises/echo-function-file-output/evaluation/contents.txt

@@ -0,0 +1 @@
+Hallo

tests/exercises/echo-function-file-output/evaluation/one.yaml

@@ -0,0 +1,7 @@
+- tab: "Test"
+ testcases:
+ - statement: echo_function("result.txt", "Hallo")
+ file:
+ content: "contents.txt"
+ location: "result.txt"
+ oracle: builtin

tests/exercises/echo-function-file-output/solution/correct.c

@@ -0,0 +1,7 @@
+#include <stdio.h>
+
+void echo_function(const char *filename, const char *string) {
+ FILE *file = fopen(filename, "w");
+ fprintf(file, "%s\n", string);
+ fclose(file);
+}

tests/exercises/echo-function-file-output/solution/correct.cs

@@ -0,0 +1,12 @@
+using System.IO;
+
+class Submission
+{
+ public static void EchoFunction(string filename, string stringToWrite)
+ {
+ using (StreamWriter writer = new StreamWriter(filename))
+ {
+ writer.WriteLine(stringToWrite);
+ }
+ }
+}

tests/exercises/echo-function-file-output/solution/correct.hs

@@ -0,0 +1,7 @@
+import System.IO (writeFile)
+
+echoFunction :: FilePath -> String -> IO ()
+echoFunction filename content = do
+ let contentWithNewline = content ++ "\n"
+ writeFile filename contentWithNewline
+ return ()

tests/exercises/echo-function-file-output/solution/correct.java

@@ -0,0 +1,15 @@
+import java.io.FileWriter;
+import java.io.IOException;
+
+public class Submission {
+
+ public static void echoFunction(String filename, String stringToWrite) {
+ try (FileWriter writer = new FileWriter(filename)) {
+ writer.write(stringToWrite);
+ writer.write(System.lineSeparator()); // Add a newline
+ } catch (IOException e) {
+ System.err.println("Error writing to file: " + e.getMessage());
+ }
+ }
+}
+