From a32ea307f844a277b84bd81b9b5a682920798c4e Mon Sep 17 00:00:00 2001 From: Niko Strijbol Date: Tue, 22 Aug 2023 17:18:19 +0200 Subject: [PATCH 1/3] Disallow additional properties in DSL --- tested/dsl/schema.json | 77 +++++++++++++++++++--------------- tested/dsl/translate_parser.py | 6 +-- tests/test_dsl_yaml.py | 17 +++++++- 3 files changed, 61 insertions(+), 39 deletions(-) diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index 9df13fa7..a6c02a40 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -1,46 +1,47 @@ { "$id" : "https://github.com/dodona-edu/universal-judge/blob/master/tested/dsl/schema.yaml", - "$schema" : "http://json-schema.org/draft-07/schema#", + "$schema" : "https://json-schema.org/draft/2019-09/schema", "title" : "DSL Schema", "description" : "DSL test suite for TESTed", "oneOf" : [ { - "$ref" : "#/definitions/_rootObject" + "$ref" : "#/$defs/_rootObject" }, { - "$ref" : "#/definitions/unit" + "$ref" : "#/$defs/unit" }, { - "$ref" : "#/definitions/_unitList" + "$ref" : "#/$defs/_unitList" } ], - "definitions" : { + "$defs" : { "_unitList" : { "type" : "array", "minItems" : 1, "items" : { - "$ref" : "#/definitions/unit" + "$ref" : "#/$defs/unit" } }, "_testcaseList" : { "type" : "array", "minItems" : 1, "items" : { - "$ref" : "#/definitions/testcase" + "$ref" : "#/$defs/testcase" } }, "_scriptList" : { "type" : "array", "minItems" : 1, "items" : { - "$ref" : "#/definitions/script" + "$ref" : "#/$defs/script" } }, "_rootObject" : { "type" : "object", + "unevaluatedProperties" : false, "properties" : { "config" : { - "$ref" : "#/definitions/globalConfig", + "$ref" : "#/$defs/globalConfig", "description" : "Configuration applicable to the whole test suite." }, "namespace" : { @@ -48,16 +49,16 @@ "description" : "Namespace of the submitted solution, in `snake_case`" }, "tabs" : { - "$ref" : "#/definitions/_unitList" + "$ref" : "#/$defs/_unitList" }, "units" : { - "$ref" : "#/definitions/_unitList" + "$ref" : "#/$defs/_unitList" }, "language" : { "description" : "Indicate that all code is in a specific language.", "oneOf" : [ { - "$ref" : "#/definitions/supportedLanguage" + "$ref" : "#/$defs/supportedLanguage" }, { "const" : "tested" @@ -81,9 +82,10 @@ }, "unit" : { "type" : "object", + "unevaluatedProperties" : false, "properties" : { "config" : { - "$ref" : "#/definitions/globalConfig", + "$ref" : "#/$defs/globalConfig", "description" : "Configuration applicable to this unit/tab" }, "hidden" : { @@ -99,16 +101,16 @@ "description" : "The name of this tab." }, "cases" : { - "$ref" : "#/definitions/_testcaseList" + "$ref" : "#/$defs/_testcaseList" }, "contexts" : { - "$ref" : "#/definitions/_testcaseList" + "$ref" : "#/$defs/_testcaseList" }, "scripts" : { - "$ref" : "#/definitions/_scriptList" + "$ref" : "#/$defs/_scriptList" }, "testcases" : { - "$ref" : "#/definitions/_scriptList" + "$ref" : "#/$defs/_scriptList" } }, "oneOf" : [ @@ -150,9 +152,10 @@ }, "testcase" : { "type" : "object", + "unevaluatedProperties" : false, "properties" : { "config" : { - "$ref" : "#/definitions/globalConfig", + "$ref" : "#/$defs/globalConfig", "description" : "Configuration settings at context level" }, "context" : { @@ -160,10 +163,10 @@ "description" : "Description of this context." }, "testcases" : { - "$ref" : "#/definitions/_scriptList" + "$ref" : "#/$defs/_scriptList" }, "script" : { - "$ref" : "#/definitions/_scriptList" + "$ref" : "#/$defs/_scriptList" } }, "oneOf" : [ @@ -181,6 +184,7 @@ }, "script" : { "type" : "object", + "unevaluatedProperties" : false, "description" : "An individual test for a statement or expression", "properties" : { "stdin" : { @@ -206,11 +210,11 @@ }, "statement" : { "description" : "The statement to evaluate.", - "$ref" : "#/definitions/expressionOrStatement" + "$ref" : "#/$defs/expressionOrStatement" }, "expression" : { "description" : "The expression to evaluate.", - "$ref" : "#/definitions/expressionOrStatement" + "$ref" : "#/$defs/expressionOrStatement" }, "exception" : { "description" : "Expected exception message", @@ -230,7 +234,7 @@ "types" : { "type" : "object", "propertyNames" : { - "$ref" : "#/definitions/supportedLanguage" + "$ref" : "#/$defs/supportedLanguage" }, "items" : { "type" : "string" @@ -243,7 +247,7 @@ "files" : { "type" : "array", "items" : { - "$ref" : "#/definitions/file" + "$ref" : "#/$defs/file" } }, "return" : { @@ -251,21 +255,21 @@ }, "return_raw" : { "description" : "Value string to parse to the expected return value", - "$ref" : "#/definitions/advancedValueOutputChannel" + "$ref" : "#/$defs/advancedValueOutputChannel" }, "stderr" : { "description" : "Expected output at stderr", - "$ref" : "#/definitions/textOutputChannel" + "$ref" : "#/$defs/textOutputChannel" }, "stdout" : { "description" : "Expected output at stdout", - "$ref" : "#/definitions/textOutputChannel" + "$ref" : "#/$defs/textOutputChannel" }, "config" : { - "$ref" : "#/definitions/globalConfig", + "$ref" : "#/$defs/globalConfig", "description" : "Configuration settings at testcase level" }, - "exitCode" : { + "exit_code" : { "type" : "integer", "description" : "Expected exit code for the run" } @@ -283,7 +287,7 @@ ] }, { - "$ref" : "#/definitions/advancedTextOutputChannel" + "$ref" : "#/$defs/advancedTextOutputChannel" } ] }, @@ -295,7 +299,7 @@ { "type" : "object", "propertyNames" : { - "$ref" : "#/definitions/supportedLanguage" + "$ref" : "#/$defs/supportedLanguage" }, "items" : { "type" : "string", @@ -306,6 +310,7 @@ }, "advancedTextOutputChannel" : { "type" : "object", + "unevaluatedProperties" : false, "description" : "Advanced output for a text output channel, such as stdout or stderr.", "required" : [ "data" @@ -321,7 +326,7 @@ ] }, "config" : { - "$ref" : "#/definitions/textConfigurationOptions" + "$ref" : "#/$defs/textConfigurationOptions" } }, "oneOf" : [ @@ -380,6 +385,7 @@ }, { "type" : "object", + "unevaluatedProperties" : false, "description" : "A custom check function.", "required" : [ "value" @@ -444,6 +450,7 @@ "type" : "object", "description" : "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", "minProperties" : 1, + "unevaluatedProperties" : false, "properties" : { "applyRounding" : { "description" : "Apply rounding when comparing as float", @@ -475,18 +482,20 @@ "type" : "object", "description" : "Global configuration properties", "minProperties" : 1, + "unevaluatedProperties" : false, "properties" : { "stdout" : { - "$ref" : "#/definitions/textConfigurationOptions" + "$ref" : "#/$defs/textConfigurationOptions" }, "stderr" : { - "$ref" : "#/definitions/textConfigurationOptions" + "$ref" : "#/$defs/textConfigurationOptions" } } }, "file" : { "type" : "object", "description" : "Path to a file for input.", + "unevaluatedProperties" : false, "required" : [ "name", "url" diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 9f31c4b1..3c29fc47 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -17,8 +17,8 @@ import yaml from attrs import define, evolve -from jsonschema import Draft7Validator from jsonschema.exceptions import ValidationError +from jsonschema.validators import Draft201909Validator from tested.datatypes import ( AdvancedNumericTypes, @@ -116,8 +116,8 @@ def _load_schema_validator(): path_to_schema = Path(__file__).parent / "schema.json" with open(path_to_schema, "r") as schema_file: schema_object = json.load(schema_file) - Draft7Validator.check_schema(schema_object) - return Draft7Validator(schema_object) + Draft201909Validator.check_schema(schema_object) + return Draft201909Validator(schema_object) _SCHEMA_VALIDATOR = _load_schema_validator() diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index 025b542f..9a177ba5 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -17,7 +17,7 @@ SequenceTypes, StringTypes, ) -from tested.dsl import translate_to_test_suite +from tested.dsl import parse_dsl, translate_to_test_suite from tested.serialisation import ( Assignment, FunctionCall, @@ -40,7 +40,6 @@ def test_parse_one_tab_ctx(): yaml_str = """ -disableOptimizations: true namespace: "solution" tabs: - tab: "Ctx" @@ -959,3 +958,17 @@ def test_old_and_new_names_work(old, new): new_suite = parse_test_suite(new_json) assert old_suite == new_suite + + +def test_additional_properties_are_not_allowed(): + yaml_str = """ + - tab: "Feedback" + testcases: + - expression: "heir(8, 10)" + not_return: [ 10, 4, 15, 11, 7, 5, 3, 2, 16, 12, 1, 6, 13, 9, 14, 8 ] + - statement: + javascript: "hello()" + python: "hello_2()" +""" + with pytest.raises(Exception): + parse_dsl(yaml_str) From 9de386aa342dc8d82394806887dab8af00364779 Mon Sep 17 00:00:00 2001 From: Niko Strijbol Date: Tue, 22 Aug 2023 17:54:11 +0200 Subject: [PATCH 2/3] Add draft 7 back for now Not all tools support this, thus it might be useful to have. --- tested/dsl/schema_draft7.json | 521 ++++++++++++++++++++++++++++++++++ 1 file changed, 521 insertions(+) create mode 100644 tested/dsl/schema_draft7.json diff --git a/tested/dsl/schema_draft7.json b/tested/dsl/schema_draft7.json new file mode 100644 index 00000000..9df13fa7 --- /dev/null +++ b/tested/dsl/schema_draft7.json @@ -0,0 +1,521 @@ +{ + "$id" : "https://github.com/dodona-edu/universal-judge/blob/master/tested/dsl/schema.yaml", + "$schema" : "http://json-schema.org/draft-07/schema#", + "title" : "DSL Schema", + "description" : "DSL test suite for TESTed", + "oneOf" : [ + { + "$ref" : "#/definitions/_rootObject" + }, + { + "$ref" : "#/definitions/unit" + }, + { + "$ref" : "#/definitions/_unitList" + } + ], + "definitions" : { + "_unitList" : { + "type" : "array", + "minItems" : 1, + "items" : { + "$ref" : "#/definitions/unit" + } + }, + "_testcaseList" : { + "type" : "array", + "minItems" : 1, + "items" : { + "$ref" : "#/definitions/testcase" + } + }, + "_scriptList" : { + "type" : "array", + "minItems" : 1, + "items" : { + "$ref" : "#/definitions/script" + } + }, + "_rootObject" : { + "type" : "object", + "properties" : { + "config" : { + "$ref" : "#/definitions/globalConfig", + "description" : "Configuration applicable to the whole test suite." + }, + "namespace" : { + "type" : "string", + "description" : "Namespace of the submitted solution, in `snake_case`" + }, + "tabs" : { + "$ref" : "#/definitions/_unitList" + }, + "units" : { + "$ref" : "#/definitions/_unitList" + }, + "language" : { + "description" : "Indicate that all code is in a specific language.", + "oneOf" : [ + { + "$ref" : "#/definitions/supportedLanguage" + }, + { + "const" : "tested" + } + ], + "default" : "tested" + } + }, + "oneOf" : [ + { + "required" : [ + "tabs" + ] + }, + { + "required" : [ + "units" + ] + } + ] + }, + "unit" : { + "type" : "object", + "properties" : { + "config" : { + "$ref" : "#/definitions/globalConfig", + "description" : "Configuration applicable to this unit/tab" + }, + "hidden" : { + "type" : "boolean", + "description" : "Defines if the unit/tab is hidden for the student or not" + }, + "unit" : { + "type" : "string", + "description" : "The name of this unit." + }, + "tab" : { + "type" : "string", + "description" : "The name of this tab." + }, + "cases" : { + "$ref" : "#/definitions/_testcaseList" + }, + "contexts" : { + "$ref" : "#/definitions/_testcaseList" + }, + "scripts" : { + "$ref" : "#/definitions/_scriptList" + }, + "testcases" : { + "$ref" : "#/definitions/_scriptList" + } + }, + "oneOf" : [ + { + "required" : [ + "tab" + ], + "oneOf" : [ + { + "required" : [ + "contexts" + ] + }, + { + "required" : [ + "testcases" + ] + } + ] + }, + { + "required" : [ + "unit" + ], + "oneOf" : [ + { + "required" : [ + "cases" + ] + }, + { + "required" : [ + "scripts" + ] + } + ] + } + ] + }, + "testcase" : { + "type" : "object", + "properties" : { + "config" : { + "$ref" : "#/definitions/globalConfig", + "description" : "Configuration settings at context level" + }, + "context" : { + "type" : "string", + "description" : "Description of this context." + }, + "testcases" : { + "$ref" : "#/definitions/_scriptList" + }, + "script" : { + "$ref" : "#/definitions/_scriptList" + } + }, + "oneOf" : [ + { + "required" : [ + "testcases" + ] + }, + { + "required" : [ + "script" + ] + } + ] + }, + "script" : { + "type" : "object", + "description" : "An individual test for a statement or expression", + "properties" : { + "stdin" : { + "description" : "Stdin for this context", + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + }, + "arguments" : { + "type" : "array", + "description" : "Array of program call arguments", + "items" : { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + } + }, + "statement" : { + "description" : "The statement to evaluate.", + "$ref" : "#/definitions/expressionOrStatement" + }, + "expression" : { + "description" : "The expression to evaluate.", + "$ref" : "#/definitions/expressionOrStatement" + }, + "exception" : { + "description" : "Expected exception message", + "oneOf" : [ + { + "type" : "string" + }, + { + "type" : "object", + "required" : [ + "types" + ], + "properties" : { + "message" : { + "type" : "string" + }, + "types" : { + "type" : "object", + "propertyNames" : { + "$ref" : "#/definitions/supportedLanguage" + }, + "items" : { + "type" : "string" + } + } + } + } + ] + }, + "files" : { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + }, + "return" : { + "description" : "Expected return value" + }, + "return_raw" : { + "description" : "Value string to parse to the expected return value", + "$ref" : "#/definitions/advancedValueOutputChannel" + }, + "stderr" : { + "description" : "Expected output at stderr", + "$ref" : "#/definitions/textOutputChannel" + }, + "stdout" : { + "description" : "Expected output at stdout", + "$ref" : "#/definitions/textOutputChannel" + }, + "config" : { + "$ref" : "#/definitions/globalConfig", + "description" : "Configuration settings at testcase level" + }, + "exitCode" : { + "type" : "integer", + "description" : "Expected exit code for the run" + } + } + }, + "textOutputChannel" : { + "anyOf" : [ + { + "description" : "A simple value which is converted into a string.", + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "$ref" : "#/definitions/advancedTextOutputChannel" + } + ] + }, + "expressionOrStatement" : { + "oneOf" : [ + { + "type" : "string" + }, + { + "type" : "object", + "propertyNames" : { + "$ref" : "#/definitions/supportedLanguage" + }, + "items" : { + "type" : "string", + "description" : "A language-specific literal which will be used verbatim." + } + } + ] + }, + "advancedTextOutputChannel" : { + "type" : "object", + "description" : "Advanced output for a text output channel, such as stdout or stderr.", + "required" : [ + "data" + ], + "properties" : { + "data" : { + "description" : "The expected data types.", + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + }, + "config" : { + "$ref" : "#/definitions/textConfigurationOptions" + } + }, + "oneOf" : [ + { + "properties" : { + "oracle" : { + "type" : "string", + "enum" : [ + "builtin" + ] + } + } + }, + { + "required" : [ + "oracle", + "language", + "file" + ], + "properties" : { + "oracle" : { + "type" : "string", + "enum" : [ + "custom_check" + ] + }, + "language" : { + "type" : "string", + "description" : "The programming language of the custom check function." + }, + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + }, + "arguments" : { + "type" : "array", + "description" : "List of 'Python' values to use as arguments to the function.", + "items" : { + "type" : "string" + } + } + } + } + ] + }, + "advancedValueOutputChannel" : { + "oneOf" : [ + { + "type" : "string", + "description" : "A 'Python' value to parse and use as the expected type." + }, + { + "type" : "object", + "description" : "A custom check function.", + "required" : [ + "value" + ], + "properties" : { + "value" : { + "type" : "string", + "description" : "The expected value." + } + }, + "oneOf" : [ + { + "properties" : { + "oracle" : { + "type" : "string", + "enum" : [ + "builtin" + ] + } + } + }, + { + "required" : [ + "oracle", + "language", + "file" + ], + "properties" : { + "oracle" : { + "type" : "string", + "enum" : [ + "custom_check" + ] + }, + "language" : { + "type" : "string", + "description" : "The programming language of the custom check function." + }, + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + }, + "arguments" : { + "type" : "array", + "description" : "List of 'Python' values to use as arguments to the function.", + "items" : { + "type" : "string" + } + } + } + } + ] + } + ] + }, + "textConfigurationOptions" : { + "type" : "object", + "description" : "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", + "minProperties" : 1, + "properties" : { + "applyRounding" : { + "description" : "Apply rounding when comparing as float", + "type" : "boolean" + }, + "caseInsensitive" : { + "description" : "Ignore case when comparing strings", + "type" : "boolean" + }, + "ignoreWhitespace" : { + "description" : "Ignore leading and trailing whitespace", + "type" : "boolean" + }, + "roundTo" : { + "description" : "The number of decimals to round at, when applying the rounding on floats", + "type" : "integer" + }, + "tryFloatingPoint" : { + "description" : "Try comparing text as floating point numbers", + "type" : "boolean" + }, + "hideExpected" : { + "description" : "Hide the expected value in feedback (default: false), not recommended to use!", + "type" : "boolean" + } + } + }, + "globalConfig" : { + "type" : "object", + "description" : "Global configuration properties", + "minProperties" : 1, + "properties" : { + "stdout" : { + "$ref" : "#/definitions/textConfigurationOptions" + }, + "stderr" : { + "$ref" : "#/definitions/textConfigurationOptions" + } + } + }, + "file" : { + "type" : "object", + "description" : "Path to a file for input.", + "required" : [ + "name", + "url" + ], + "properties" : { + "name" : { + "type" : "string", + "description" : "File name" + }, + "url" : { + "type" : "string", + "format" : "uri", + "description" : "Relative path to the file in the `description` folder of a Dodona exercise" + } + } + }, + "supportedLanguage" : { + "type" : "string", + "enum" : [ + "bash", + "c", + "haskell", + "java", + "javascript", + "kotlin", + "python", + "runhaskell", + "csharp" + ] + } + } +} From 86d4d0333d7643d9645bdbc0aba44ce0642b64a3 Mon Sep 17 00:00:00 2001 From: Niko Strijbol Date: Tue, 22 Aug 2023 18:05:34 +0200 Subject: [PATCH 3/3] Fix test suites --- tests/exercises/objects/evaluation/missing_key_types.yaml | 2 +- tests/exercises/objects/evaluation/no-test.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/exercises/objects/evaluation/missing_key_types.yaml b/tests/exercises/objects/evaluation/missing_key_types.yaml index 37ba66a4..ccd82228 100644 --- a/tests/exercises/objects/evaluation/missing_key_types.yaml +++ b/tests/exercises/objects/evaluation/missing_key_types.yaml @@ -1,4 +1,4 @@ - tab: "Feedback" testcases: - expression: '{{"a"}: [int32(1)], {"b"}: "a.txt"}' - return-raw: '{{"a"}: [int32(1)], {"b"}: "a.txt"}' + return_raw: '{{"a"}: [int32(1)], {"b"}: "a.txt"}' diff --git a/tests/exercises/objects/evaluation/no-test.yaml b/tests/exercises/objects/evaluation/no-test.yaml index 4b876b51..585abb16 100644 --- a/tests/exercises/objects/evaluation/no-test.yaml +++ b/tests/exercises/objects/evaluation/no-test.yaml @@ -1,11 +1,11 @@ - tab: "Feedback" testcases: - statement: '{["a", "b"], ["c"]}' - return-raw: '{["a", "b"], ["a"]}' + return_raw: '{["a", "b"], ["a"]}' - statement: 'x = {{"a"}: [int16(1)], {"b"}: [int16(0)]}' - statement: 'x = {{"a"}: [int32(1)], {"b"}: "a"}' - expression: '{{"a"}: [int32(1)], {"b"}: "a.txt"}' - return-raw: '{{"a"}: [int32(1)], {"b"}: "a.txt"}' + return_raw: '{{"a"}: [int32(1)], {"b"}: "a.txt"}' files: - name: "a.txt" url: "a.txt"