From 85857f1dcb41ca59a250f1547906c80a0b7ab0aa Mon Sep 17 00:00:00 2001 From: Niko Strijbol Date: Thu, 12 Sep 2024 23:12:24 +0200 Subject: [PATCH] Re-add config inheritance --- tested/dsl/schema-strict.json | 56 ++++++++++++++++------ tested/dsl/schema.json | 56 ++++++++++++++++------ tested/dsl/translate_parser.py | 76 +++++++++++++++++++++-------- tested/utils.py | 1 - tests/test_dsl_yaml.py | 88 +++++++++++++++++++++++++++++++++- 5 files changed, 225 insertions(+), 52 deletions(-) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index 397f2110..f9bd7a34 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -70,6 +70,9 @@ "definitions" : { "description" : "Define hashes to use elsewhere.", "type" : "object" + }, + "config": { + "$ref": "#/definitions/inheritableConfigObject" } } }, @@ -111,6 +114,9 @@ "definitions" : { "description" : "Define objects to use elsewhere.", "type" : "object" + }, + "config": { + "$ref": "#/definitions/inheritableConfigObject" } }, "oneOf" : [ @@ -160,6 +166,9 @@ "definitions" : { "description" : "Define objects to use elsewhere.", "type" : "object" + }, + "config": { + "$ref": "#/definitions/inheritableConfigObject" } }, "oneOf" : [ @@ -585,21 +594,7 @@ "const" : "builtin" }, "config" : { - "anyOf" : [ - { - "$ref" : "#/definitions/textConfigurationOptions" - }, - { - "type" : "object", - "properties" : { - "mode": { - "type" : "string", - "enum" : ["full", "line"], - "default" : "full" - } - } - } - ] + "$ref" : "#/definitions/fileConfigurationOptions" } } }, @@ -845,6 +840,23 @@ } } }, + "fileConfigurationOptions": { + "anyOf" : [ + { + "$ref" : "#/definitions/textConfigurationOptions" + }, + { + "type" : "object", + "properties" : { + "mode": { + "type" : "string", + "enum" : ["full", "line"], + "default" : "full" + } + } + } + ] + }, "textualType" : { "description" : "Simple textual value, converted to string.", "type" : [ @@ -862,6 +874,20 @@ "expression" ] } + }, + "inheritableConfigObject": { + "type": "object", + "properties" : { + "stdout": { + "$ref" : "#/definitions/textConfigurationOptions" + }, + "stderr": { + "$ref" : "#/definitions/textConfigurationOptions" + }, + "file": { + "$ref" : "#/definitions/fileConfigurationOptions" + } + } } } } diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index e9896ff6..2f714949 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -70,6 +70,9 @@ "definitions" : { "description" : "Define hashes to use elsewhere.", "type" : "object" + }, + "config": { + "$ref": "#/definitions/inheritableConfigObject" } } }, @@ -111,6 +114,9 @@ "definitions" : { "description" : "Define objects to use elsewhere.", "type" : "object" + }, + "config": { + "$ref": "#/definitions/inheritableConfigObject" } }, "oneOf" : [ @@ -160,6 +166,9 @@ "definitions" : { "description" : "Define objects to use elsewhere.", "type" : "object" + }, + "config": { + "$ref": "#/definitions/inheritableConfigObject" } }, "oneOf" : [ @@ -585,21 +594,7 @@ "const" : "builtin" }, "config" : { - "anyOf" : [ - { - "$ref" : "#/definitions/textConfigurationOptions" - }, - { - "type" : "object", - "properties" : { - "mode": { - "type" : "string", - "enum" : ["full", "line"], - "default" : "full" - } - } - } - ] + "$ref" : "#/definitions/fileConfigurationOptions" } } }, @@ -845,6 +840,23 @@ } } }, + "fileConfigurationOptions": { + "anyOf" : [ + { + "$ref" : "#/definitions/textConfigurationOptions" + }, + { + "type" : "object", + "properties" : { + "mode": { + "type" : "string", + "enum" : ["full", "line"], + "default" : "full" + } + } + } + ] + }, "textualType" : { "description" : "Simple textual value, converted to string.", "type" : [ @@ -856,6 +868,20 @@ }, "yamlValue" : { "description" : "A value represented as YAML." + }, + "inheritableConfigObject": { + "type": "object", + "properties" : { + "stdout": { + "$ref" : "#/definitions/textConfigurationOptions" + }, + "stderr": { + "$ref" : "#/definitions/textConfigurationOptions" + }, + "file": { + "$ref" : "#/definitions/fileConfigurationOptions" + } + } } } } diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index fa7d4e02..9a56cca0 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -7,7 +7,7 @@ from typing import Any, Literal, Type, TypeVar, cast import yaml -from attrs import define, evolve +from attrs import define, evolve, field from jsonschema import TypeChecker from jsonschema.exceptions import ValidationError from jsonschema.protocols import Validator @@ -69,7 +69,7 @@ TextOutputChannel, ValueOutputChannel, ) -from tested.utils import get_args +from tested.utils import get_args, recursive_dict_merge YamlDict = dict[str, "YamlObject"] @@ -229,9 +229,16 @@ class InvalidYamlError(ValueError): class DslContext: """ Carries context in each level. + + This function will, in essence, make two properties inheritable from the global + and tab context: + + - The "config" property, which has config for "stdout", "stderr", and "file". + - The "files" property, which is a list of files. """ - files: list[FileUrl] + files: list[FileUrl] = field(factory=list) + config: dict[str, dict] = field(factory=dict) language: SupportedLanguage | Literal["tested"] = "tested" def deepen_context(self, new_level: YamlDict | None) -> "DslContext": @@ -246,13 +253,28 @@ def deepen_context(self, new_level: YamlDict | None) -> "DslContext": if new_level is None: return self - new_files = self.files + the_files = self.files if "files" in new_level: assert isinstance(new_level["files"], list) additional_files = {_convert_file(f) for f in new_level["files"]} - new_files = list(set(self.files) | additional_files) + the_files = list(set(self.files) | additional_files) + + the_config = self.config + if "config" in new_level: + assert isinstance(new_level["config"], dict) + the_config = recursive_dict_merge(the_config, new_level["config"]) - return evolve(self, files=new_files) + return evolve(self, files=the_files, config=the_config) + + def merge_inheritable_with_specific_config( + self, level: YamlDict, config_name: str + ) -> dict: + inherited_options = self.config.get(config_name, dict()) + specific_options = level.get("config", dict()) + assert isinstance( + specific_options, dict + ), f"The config options for {config_name} must be a dictionary, not a {type(specific_options)}" + return recursive_dict_merge(inherited_options, specific_options) def convert_validation_error_to_group( @@ -418,36 +440,50 @@ def _convert_language_specific_oracle(stream: dict) -> LanguageSpecificOracle: return LanguageSpecificOracle(functions=the_functions, arguments=the_args) -def _convert_text_output_channel(stream: YamlObject) -> TextOutputChannel: +def _convert_text_output_channel( + stream: YamlObject, context: DslContext, config_name: str +) -> TextOutputChannel: + # Get the config applicable to this level. + # Either attempt to get it from an object, or using the inherited options as is. + if isinstance(stream, str): + config = context.config.get(config_name, dict()) + raw_data = stream + else: + assert isinstance(stream, dict) + config = context.merge_inheritable_with_specific_config(stream, config_name) + raw_data = str(stream["data"]) + + # Normalize the data if necessary. + if config.get("normalizeTrailingNewlines", True): + data = _ensure_trailing_newline(raw_data) + else: + data = raw_data + if isinstance(stream, str): - data = _ensure_trailing_newline(stream) - return TextOutputChannel(data=data, oracle=GenericTextOracle()) + return TextOutputChannel(data=data, oracle=GenericTextOracle(options=config)) else: assert isinstance(stream, dict) - data = str(stream["data"]) if "oracle" not in stream or stream["oracle"] == "builtin": - config = cast(dict, stream.get("config", {})) - if config.get("normalizeTrailingNewlines", True): - data = _ensure_trailing_newline(data) return TextOutputChannel( data=data, oracle=GenericTextOracle(options=config) ) elif stream["oracle"] == "custom_check": - data = _ensure_trailing_newline(data) return TextOutputChannel( data=data, oracle=_convert_custom_check_oracle(stream) ) raise TypeError(f"Unknown text oracle type: {stream['oracle']}") -def _convert_file_output_channel(stream: YamlObject) -> FileOutputChannel: +def _convert_file_output_channel( + stream: YamlObject, context: DslContext, config_name: str +) -> FileOutputChannel: assert isinstance(stream, dict) expected = str(stream["content"]) actual = str(stream["location"]) if "oracle" not in stream or stream["oracle"] == "builtin": - config = cast(dict, stream.get("config", {})) + config = context.merge_inheritable_with_specific_config(stream, config_name) if "mode" not in config: config["mode"] = "full" @@ -566,11 +602,11 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: output.result = return_channel if (stdout := testcase.get("stdout")) is not None: - output.stdout = _convert_text_output_channel(stdout) + output.stdout = _convert_text_output_channel(stdout, context, "stdout") if (file := testcase.get("file")) is not None: - output.file = _convert_file_output_channel(file) + output.file = _convert_file_output_channel(file, context, "file") if (stderr := testcase.get("stderr")) is not None: - output.stderr = _convert_text_output_channel(stderr) + output.stderr = _convert_text_output_channel(stderr, context, "stderr") if (exception := testcase.get("exception")) is not None: if isinstance(exception, str): message = exception @@ -690,7 +726,7 @@ def _convert_dsl(dsl_object: YamlObject) -> Suite: :param dsl_object: A validated DSL test suite object. :return: A full test suite. """ - context = DslContext(files=[]) + context = DslContext() if isinstance(dsl_object, list): namespace = None tab_list = dsl_object diff --git a/tested/utils.py b/tested/utils.py index 20bb1aad..e913c8b6 100644 --- a/tested/utils.py +++ b/tested/utils.py @@ -148,7 +148,6 @@ def recursive_dict_merge(one: dict, two: dict) -> dict: """ new_dictionary = {} - # noinspection PyTypeChecker for key, value in one.items(): new_dictionary[key] = value diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index 0f45d701..3caef72d 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -430,7 +430,7 @@ def test_statement_with_yaml_dict(): assert isinstance(test.output.result.value, ObjectType) -def test_global_config_trickles_down(): +def test_global_definition_config_trickles_down(): yaml_str = """ definitions: config: &stdout @@ -462,6 +462,92 @@ def test_global_config_trickles_down(): assert config["caseInsensitive"] +def test_global_config_trickles_down(): + yaml_str = """ +config: + stdout: + applyRounding: true + roundTo: 63 + tryFloatingPoint: true + caseInsensitive: true + namespace: "solution" +tabs: +- tab: "Ctx" + hidden: true + testcases: + - arguments: [ "--arg", "argument" ] + stdin: "Input string" + stdout: "Output string" + stderr: "Error string" + exit_code: 1 + """ + json_str = translate_to_test_suite(yaml_str) + suite = parse_test_suite(json_str) + stdout = suite.tabs[0].contexts[0].testcases[0].output.stdout + assert isinstance(stdout.oracle, GenericTextOracle) + config = stdout.oracle.options + assert config["applyRounding"] + assert config["roundTo"] == 63 + assert config["tryFloatingPoint"] + assert config["caseInsensitive"] + + +def test_tab_config_trickles_down_stdout(): + yaml_str = """ +- tab: "Ctx" + config: + stdout: + applyRounding: true + roundTo: 63 + tryFloatingPoint: true + caseInsensitive: true + namespace: "solution" + testcases: + - arguments: [ "--arg", "argument" ] + stdin: "Input string" + stdout: "Output string" + stderr: "Error string" + exit_code: 1 + """ + json_str = translate_to_test_suite(yaml_str) + suite = parse_test_suite(json_str) + stdout = suite.tabs[0].contexts[0].testcases[0].output.stdout + assert isinstance(stdout.oracle, GenericTextOracle) + config = stdout.oracle.options + assert config["applyRounding"] + assert config["roundTo"] == 63 + assert config["tryFloatingPoint"] + assert config["caseInsensitive"] + + +def test_tab_config_trickles_down_stderr(): + yaml_str = """ +- tab: "Ctx" + config: + stderr: + applyRounding: true + roundTo: 63 + tryFloatingPoint: true + caseInsensitive: true + namespace: "solution" + testcases: + - arguments: [ "--arg", "argument" ] + stdin: "Input string" + stdout: "Output string" + stderr: "Error string" + exit_code: 1 + """ + json_str = translate_to_test_suite(yaml_str) + suite = parse_test_suite(json_str) + stderr = suite.tabs[0].contexts[0].testcases[0].output.stderr + assert isinstance(stderr.oracle, GenericTextOracle) + config = stderr.oracle.options + assert config["applyRounding"] + assert config["roundTo"] == 63 + assert config["tryFloatingPoint"] + assert config["caseInsensitive"] + + def test_expression_raw_return(): yaml_str = """ - tab: 'Test'