From 88c3653f969ccafa6438fa683fa7e5baf559c014 Mon Sep 17 00:00:00 2001 From: Niko Strijbol Date: Mon, 7 Aug 2023 16:04:56 +0200 Subject: [PATCH 1/2] Fix type hints for type checker --- flake.nix | 10 +- flake.old.nix | 84 --------- tested/datatypes/__init__.py | 8 +- tested/description_instance.py | 14 +- tested/dodona.py | 8 +- tested/dsl/ast_translator.py | 44 +++-- tested/dsl/translate_parser.py | 47 +++-- tested/evaluators/__init__.py | 13 +- tested/evaluators/common.py | 14 +- tested/evaluators/exception.py | 12 +- tested/evaluators/exitcode.py | 4 +- tested/evaluators/ignored.py | 4 +- tested/evaluators/nothing.py | 4 +- tested/evaluators/programmed.py | 40 ++--- tested/evaluators/specific.py | 20 +-- tested/evaluators/text.py | 2 +- tested/evaluators/value.py | 200 ++++++++++++++-------- tested/features.py | 20 ++- tested/instantiate_exercise.py | 16 +- tested/judge/collector.py | 11 +- tested/judge/compilation.py | 4 +- tested/judge/core.py | 11 +- tested/judge/evaluation.py | 36 ++-- tested/judge/execution.py | 11 +- tested/judge/programmed.py | 35 ++-- tested/judge/utils.py | 4 +- tested/languages/bash/config.py | 1 + tested/languages/bash/generators.py | 31 ++-- tested/languages/bash/linter.py | 16 +- tested/languages/c/config.py | 6 +- tested/languages/c/generators.py | 16 +- tested/languages/c/linter.py | 2 +- tested/languages/config.py | 16 +- tested/languages/conventionalize.py | 1 + tested/languages/csharp/config.py | 3 +- tested/languages/csharp/generators.py | 35 ++-- tested/languages/description_generator.py | 21 ++- tested/languages/generation.py | 17 +- tested/languages/haskell/config.py | 8 +- tested/languages/haskell/generators.py | 23 ++- tested/languages/haskell/linter.py | 16 +- tested/languages/java/config.py | 6 +- tested/languages/java/generators.py | 36 ++-- tested/languages/java/linter.py | 7 +- tested/languages/javascript/config.py | 21 ++- tested/languages/javascript/generators.py | 27 ++- tested/languages/javascript/linter.py | 9 +- tested/languages/kotlin/config.py | 15 +- tested/languages/kotlin/generators.py | 32 +++- tested/languages/kotlin/linter.py | 24 +-- tested/languages/preparation.py | 22 ++- tested/languages/python/config.py | 10 +- tested/languages/python/generators.py | 23 ++- tested/languages/python/linter.py | 5 +- tested/languages/runhaskell/config.py | 2 + tested/languages/utils.py | 6 +- tested/serialisation.py | 112 ++++++++---- tested/testsuite.py | 54 +++--- tested/utils.py | 57 +----- tests/test_dsl_yaml.py | 2 - tests/test_evaluators.py | 62 ++++++- tests/test_functionality.py | 2 +- tests/test_serialisation.py | 8 +- 63 files changed, 826 insertions(+), 604 deletions(-) delete mode 100644 flake.old.nix diff --git a/flake.nix b/flake.nix index 403eb317..bc051b36 100644 --- a/flake.nix +++ b/flake.nix @@ -69,15 +69,17 @@ typing-inspect pyyaml pygments - python-i18n - # For Pycharm - setuptools + python-i18n ]; python-env = python.withPackages(ps: (core-packages ps) ++ [ ps.pylint ps.pytest ps.pytest-mock ps.pytest-cov + # For Pycharm + ps.setuptools + ps.isort + ps.black ]); core-deps = [ (python.withPackages(ps: (core-packages ps) ++ [ps.pylint])) @@ -115,7 +117,7 @@ default = tested; tested = pkgs.devshell.mkShell { name = "TESTed"; - packages = [python-env] ++ haskell-deps ++ node-deps ++ bash-deps ++ c-deps ++ java-deps ++ kotlin-deps ++ csharp-deps; + packages = [python-env pkgs.nodePackages.pyright] ++ haskell-deps ++ node-deps ++ bash-deps ++ c-deps ++ java-deps ++ kotlin-deps ++ csharp-deps; devshell.startup.link.text = '' mkdir -p "$PRJ_DATA_DIR/current" ln -sfn "${python-env}/${python-env.sitePackages}" "$PRJ_DATA_DIR/current/python-packages" diff --git a/flake.old.nix b/flake.old.nix deleted file mode 100644 index f4611276..00000000 --- a/flake.old.nix +++ /dev/null @@ -1,84 +0,0 @@ -{ - description = "TESTed"; - - inputs = { - nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; - flake-utils.url = "github:numtide/flake-utils"; - devshell = { - url = "github:numtide/devshell"; - inputs = { - flake-utils.follows = "flake-utils"; - nixpkgs.follows = "nixpkgs"; - }; - }; - }; - - outputs = { self, nixpkgs, devshell, flake-utils, mach-nix, ... }: - flake-utils.lib.eachDefaultSystem (system: - let - pkgs = import nixpkgs { inherit system; overlays = [ devshell.overlays.default ]; }; - python-env = pkgs.python311; - ghc-aeson = pkgs.haskell.packages.ghc94.ghcWithPackages (p: [p.aeson]); - in - { - devShells = rec { - default = tested; - tested = pkgs.devshell.mkShell { - name = "TESTed"; - packages = with pkgs; [ - pkgs.nodejs - pkgs.nodePackages.npm - pkgs.nodePackages.eslint - ghc-aeson - python-env - pkgs.pipenv - pkgs.shellcheck - pkgs.cppcheck - pkgs.hlint - pkgs.checkstyle - pkgs.kotlin - pkgs.ktlint - pkgs.openjdk17 - pkgs.gcc - pkgs.dotnetCorePackages.sdk_6_0 - pkgs.pylint - ]; - devshell.startup.link.text = '' - mkdir -p "$PRJ_DATA_DIR/current" - ln -sfn "${python-env}/${python-env.sitePackages}" "$PRJ_DATA_DIR/current/python-packages" - ln -sfn "${python-env}" "$PRJ_DATA_DIR/current/python" - ''; - env = [ - { - name = "DOTNET_ROOT"; - eval = "${pkgs.dotnetCorePackages.sdk_6_0}"; - } - { - name = "NODE_PATH"; - prefix = "$(npm get prefix)"; - } - ]; - commands = [ - { - name = "test:stable"; - category = "tests"; - help = "Run the non-flaky tests."; - command = '' - python -m pytest tests/ -m "not flaky" - ''; - } - { - name = "test:all"; - category = "tests"; - help = "Run all tests."; - command = '' - python -m pytest tests/ - ''; - } - ]; - }; - }; - } - ); -} - diff --git a/tested/datatypes/__init__.py b/tested/datatypes/__init__.py index 482a9fba..1f1ad2fd 100644 --- a/tested/datatypes/__init__.py +++ b/tested/datatypes/__init__.py @@ -35,10 +35,10 @@ NumericTypes = Union[BasicNumericTypes, AdvancedNumericTypes] StringTypes = Union[BasicStringTypes, AdvancedStringTypes] -BooleanTypes = Union[BasicBooleanTypes] +BooleanTypes = BasicBooleanTypes NothingTypes = Union[BasicNothingTypes, AdvancedNothingTypes] SequenceTypes = Union[BasicSequenceTypes, AdvancedSequenceTypes] -ObjectTypes = Union[BasicObjectTypes] +ObjectTypes = BasicObjectTypes SimpleTypes = Union[NumericTypes, StringTypes, BooleanTypes, NothingTypes] ComplexTypes = Union[SequenceTypes, ObjectTypes] @@ -56,10 +56,10 @@ def resolve_to_basic(type_: AllTypes) -> BasicTypes: """ Resolve a type to its basic type. Basic types are returned unchanged. """ - if isinstance(type_, get_args(BasicTypes)): + if isinstance(type_, BasicTypes): return type_ - assert isinstance(type_, get_args(AdvancedTypes)) + assert isinstance(type_, AdvancedTypes) return type_.base_type diff --git a/tested/description_instance.py b/tested/description_instance.py index 48203e5c..85d3b079 100644 --- a/tested/description_instance.py +++ b/tested/description_instance.py @@ -140,14 +140,14 @@ def create_description_instance_from_template( judge_directory = Path(__file__).parent.parent global_config = GlobalConfig( dodona=DodonaConfig( - resources="", - source="", + resources="", # type: ignore + source="", # type: ignore time_limit=0, memory_limit=0, natural_language=natural_language, programming_language=programming_language, - workdir="", - judge=str(judge_directory), + workdir="", # type: ignore + judge=judge_directory, test_suite="suite.yaml", ), context_separator_secret="", @@ -185,7 +185,7 @@ def get_variable(var_name: str, is_global: bool = True): if is_html: namespace = html.escape(namespace) - return template.render( + return template.render( # type: ignore function=partial(description_generator.get_function_name, is_html=is_html), property=partial(description_generator.get_property_name, is_html=is_html), variable=get_variable, @@ -224,9 +224,9 @@ def create_description_instance( if not language_exists(programming_language): raise ValueError(f"Language {programming_language} doesn't exists") - template = prepare_template(template, is_html) + template = prepare_template(template, is_html) # type: ignore return create_description_instance_from_template( - template, programming_language, natural_language, namespace, is_html + template, programming_language, natural_language, namespace, is_html # type: ignore ) diff --git a/tested/dodona.py b/tested/dodona.py index 1c7b7dd4..6e8cbcf4 100644 --- a/tested/dodona.py +++ b/tested/dodona.py @@ -12,7 +12,7 @@ import dataclasses import json from enum import StrEnum, auto, unique -from typing import IO, Literal, Optional, Type, Union +from typing import IO, Literal, Optional, Union from pydantic import BaseModel from pydantic.dataclasses import dataclass @@ -138,7 +138,7 @@ class AnnotateCode: row: Index text: str - externalUrl: str = None + externalUrl: Optional[str] = None column: Optional[Index] = None type: Optional[Severity] = None rows: Optional[Index] = None @@ -227,7 +227,9 @@ class CloseJudgment: } -def close_for(type_: str) -> Type[Update]: +def close_for( + type_: str, +) -> type[CloseJudgment | CloseTab | CloseContext | CloseTestcase | CloseTest]: return _mapping[type_] diff --git a/tested/dsl/ast_translator.py b/tested/dsl/ast_translator.py index c95c7c7d..e2b46db6 100644 --- a/tested/dsl/ast_translator.py +++ b/tested/dsl/ast_translator.py @@ -22,14 +22,15 @@ - Collection and datastructure literals - Negation operator - Function calls -- Keyword arguments (ie. named arguments) -- Properties (ie. attributes) +- Keyword arguments (i.e. named arguments) +- Properties (i.e. attributes) """ import ast import dataclasses -from typing import Optional +from typing import Literal, Optional, cast, overload +from _decimal import Decimal from pydantic import ValidationError from tested.datatypes import ( @@ -51,6 +52,7 @@ ObjectKeyValuePair, ObjectType, SequenceType, + SpecialNumbers, Statement, Value, VariableType, @@ -70,11 +72,9 @@ def _is_and_get_allowed_empty(node: ast.Call) -> Optional[Value]: """ assert isinstance(node.func, ast.Name) if node.func.id in AdvancedSequenceTypes.__members__.values(): - # noinspection PyTypeChecker - return SequenceType(type=node.func.id, data=[]) + return SequenceType(type=cast(AdvancedSequenceTypes, node.func.id), data=[]) elif node.func.id in BasicSequenceTypes.__members__.values(): - # noinspection PyTypeChecker - return SequenceType(type=node.func.id, data=[]) + return SequenceType(type=cast(BasicSequenceTypes, node.func.id), data=[]) elif node.func.id in BasicObjectTypes.__members__.values(): return ObjectType(type=BasicObjectTypes.MAP, data=[]) else: @@ -96,6 +96,7 @@ def _is_type_cast(node: ast.expr) -> bool: def _convert_ann_assignment(node: ast.AnnAssign) -> Assignment: if not isinstance(node.target, ast.Name): raise InvalidDslError("You can only assign to simple variables") + assert node.value value = _convert_expression(node.value, False) if isinstance(node.annotation, ast.Name): type_ = node.annotation.id @@ -109,7 +110,11 @@ def _convert_ann_assignment(node: ast.AnnAssign) -> Assignment: if not is_our_type: type_ = VariableType(data=type_) - return Assignment(variable=node.target.id, expression=value, type=type_) + return Assignment( + variable=node.target.id, + expression=value, + type=cast(VariableType | AllTypes, type_), + ) def _convert_assignment(node: ast.Assign) -> Assignment: @@ -125,7 +130,7 @@ def _convert_assignment(node: ast.Assign) -> Assignment: # Support a few obvious ones, such as constructor calls or literal values. type_ = None - if isinstance(value, get_args(Value)): + if isinstance(value, Value): type_ = value.type elif isinstance(value, FunctionCall) and value.type == FunctionType.CONSTRUCTOR: type_ = VariableType(data=value.name) @@ -135,6 +140,7 @@ def _convert_assignment(node: ast.Assign) -> Assignment: f"Could not deduce the type of variable {variable.id}: add a type annotation." ) + assert isinstance(type_, AllTypes | VariableType) return Assignment(variable=variable.id, expression=value, type=type_) @@ -162,7 +168,8 @@ def _convert_call(node: ast.Call) -> FunctionCall: for keyword in node.keywords: arguments.append( NamedArgument( - name=keyword.arg, value=_convert_expression(keyword.value, False) + name=cast(str, keyword.arg), + value=_convert_expression(keyword.value, False), ) ) @@ -184,6 +191,7 @@ def _convert_constant(node: ast.Constant) -> Value: def _convert_expression(node: ast.expr, is_return: bool) -> Expression: if _is_type_cast(node): assert isinstance(node, ast.Call) + assert isinstance(node.func, ast.Name) # "Casts" of sequence types can also be used a constructor for an empty sequence. # For example, "set()", "map()", ... @@ -210,7 +218,7 @@ def _convert_expression(node: ast.expr, is_return: bool) -> Expression: subexpression = node.args[0] value = _convert_expression(subexpression, is_return) - if not isinstance(value, get_args(Value)): + if not isinstance(value, Value): raise InvalidDslError( "The argument of a cast function must resolve to a value." ) @@ -254,7 +262,8 @@ def _convert_expression(node: ast.expr, is_return: bool) -> Expression: elif isinstance(node, ast.Dict): elements = [ ObjectKeyValuePair( - _convert_expression(k, is_return), _convert_expression(v, is_return) + key=_convert_expression(cast(ast.expr, k), is_return), + value=_convert_expression(v, is_return), ) for k, v in zip(node.keys, node.values) ] @@ -269,6 +278,7 @@ def _convert_expression(node: ast.expr, is_return: bool) -> Expression: value = _convert_constant(node.operand) if not isinstance(value, NumberType): raise InvalidDslError("'-' is only supported on literal numbers") + assert isinstance(value.data, Decimal | int | float) return NumberType(type=value.type, data=-value.data) else: raise InvalidDslError(f"Unsupported expression type: {type(node)}") @@ -303,6 +313,16 @@ def _translate_to_ast(node: ast.Interactive, is_return: bool) -> Statement: return _convert_statement(statement_or_expression) +@overload +def parse_string(code: str, is_return: Literal[True]) -> Value: + ... + + +@overload +def parse_string(code: str, is_return: Literal[False] = False) -> Statement: + ... + + def parse_string(code: str, is_return=False) -> Statement: """ Parse a string with Python code into our AST. diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index ed96c92d..6cd8d073 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -1,7 +1,7 @@ import json from logging import getLogger from pathlib import Path -from typing import Callable, Dict, List, Optional, TextIO, TypeVar, Union +from typing import Callable, Dict, List, Optional, TextIO, TypeVar, Union, cast import yaml from jsonschema import Draft7Validator @@ -109,6 +109,7 @@ def _deepen_config_level( if new_level_object is None or "config" not in new_level_object: return current_level + assert isinstance(new_level_object["config"], dict) return recursive_dict_merge(current_level, new_level_object["config"]) @@ -142,6 +143,8 @@ def _convert_value(value: YamlObject) -> Value: def _convert_file(link_file: YamlDict) -> FileUrl: + assert isinstance(link_file["name"], str) + assert isinstance(link_file["url"], str) return FileUrl(name=link_file["name"], url=link_file["url"]) @@ -168,8 +171,8 @@ def _convert_text_output_channel( ) else: assert isinstance(stream, dict) + data = str(stream["data"]) if "evaluator" not in stream or stream["evaluator"] == "builtin": - data = stream["data"] existing_config = config.get(config_name, {}) config = _deepen_config_level(stream, existing_config) return TextOutputChannel( @@ -177,7 +180,7 @@ def _convert_text_output_channel( ) elif stream["evaluator"] == "custom": return TextOutputChannel( - data=stream["data"], evaluator=_convert_programmed_evaluator(stream) + data=data, evaluator=_convert_programmed_evaluator(stream) ) raise TypeError(f"Unknown text evaluator type: {stream['evaluator']}") @@ -185,15 +188,18 @@ def _convert_text_output_channel( def _convert_advanced_value_output_channel(stream: YamlObject) -> ValueOutputChannel: if isinstance(stream, str): value = parse_string(stream, is_return=True) + assert isinstance(value, Value) return ValueOutputChannel(value=value) else: assert isinstance(stream, dict) + assert isinstance(stream["value"], str) + value = parse_string(stream["value"], is_return=True) + assert isinstance(value, Value) if "evaluator" not in stream or stream["evaluator"] == "builtin": - value = parse_string(stream["value"], is_return=True) return ValueOutputChannel(value=value) elif stream["evaluator"] == "custom": return ValueOutputChannel( - value=parse_string(stream["value"], is_return=True), + value=value, evaluator=_convert_programmed_evaluator(stream), ) raise TypeError(f"Unknown value evaluator type: {stream['evaluator']}") @@ -203,14 +209,16 @@ def _convert_testcase(testcase: YamlDict, previous_config: dict) -> Testcase: config = _deepen_config_level(testcase, previous_config) if (expr_stmt := testcase.get("statement", testcase.get("expression"))) is not None: + assert isinstance(expr_stmt, str) the_input = parse_string(expr_stmt) else: - stdin = ( - TextData(data=testcase["stdin"]) - if "stdin" in testcase - else EmptyChannel.NONE - ) + if "stdin" in testcase: + assert isinstance(testcase["stdin"], str) + stdin = TextData(data=testcase["stdin"]) + else: + stdin = EmptyChannel.NONE arguments = testcase.get("arguments", []) + assert isinstance(arguments, list) the_input = MainInput(stdin=stdin, arguments=arguments) output = Output() @@ -226,12 +234,14 @@ def _convert_testcase(testcase: YamlDict, previous_config: dict) -> Testcase: else: assert isinstance(exception, dict) message = exception.get("message") - types = exception["types"] + assert isinstance(message, str) + assert isinstance(exception["types"], dict) + types = cast(Dict[str, str], exception["types"]) output.exception = ExceptionOutputChannel( exception=ExpectedException(message=message, types=types) ) if (exit_code := testcase.get("exit_code")) is not None: - output.exit_code = ExitCodeOutputChannel(exit_code) + output.exit_code = ExitCodeOutputChannel(value=cast(int, exit_code)) if (result := testcase.get("return")) is not None: if "return_raw" in testcase: raise ValueError("Both a return and return_raw value is not allowed.") @@ -244,6 +254,7 @@ def _convert_testcase(testcase: YamlDict, previous_config: dict) -> Testcase: # TODO: allow propagation of files... files = [] if "files" in testcase: + assert isinstance(testcase["files"], list) for yaml_file in testcase["files"]: files.append(_convert_file(yaml_file)) @@ -252,6 +263,7 @@ def _convert_testcase(testcase: YamlDict, previous_config: dict) -> Testcase: def _convert_context(context: YamlDict, previous_config: dict) -> Context: config = _deepen_config_level(context, previous_config) + assert isinstance(context["testcases"], list) testcases = _convert_dsl_list(context["testcases"], config, _convert_testcase) return Context(testcases=testcases) @@ -265,25 +277,26 @@ def _convert_tab(tab: YamlDict, previous_config: dict) -> Tab: :return: A full tab. """ config = _deepen_config_level(tab, previous_config) - hidden = tab.get("hidden", None) name = tab["tab"] + assert isinstance(name, str) # The tab can have testcases or contexts. if "contexts" in tab: + assert isinstance(tab["contexts"], list) contexts = _convert_dsl_list(tab["contexts"], config, _convert_context) else: - assert "testcases" in tab + assert isinstance(tab["testcases"], list) testcases = _convert_dsl_list(tab["testcases"], config, _convert_testcase) contexts = [Context(testcases=[t]) for t in testcases] - return Tab(name=name, hidden=hidden, contexts=contexts) + return Tab(name=name, contexts=contexts) T = TypeVar("T") def _convert_dsl_list( - dsl_list: list, config: dict, converter: Callable[[YamlObject, dict], T] + dsl_list: list, config: dict, converter: Callable[[YamlDict, dict], T] ) -> List[T]: """ Convert a list of YAML objects into a test suite object. @@ -319,9 +332,11 @@ def _convert_dsl(dsl_object: YamlObject) -> Suite: namespace = dsl_object.get("namespace") config = _deepen_config_level(dsl_object, {}) tab_list = dsl_object["tabs"] + assert isinstance(tab_list, list) tabs = _convert_dsl_list(tab_list, config, _convert_tab) if namespace: + assert isinstance(namespace, str) return Suite(tabs=tabs, namespace=namespace) else: return Suite(tabs=tabs) diff --git a/tested/evaluators/__init__.py b/tested/evaluators/__init__.py index 044a765b..cb9dfb28 100644 --- a/tested/evaluators/__init__.py +++ b/tested/evaluators/__init__.py @@ -24,11 +24,11 @@ def evaluate_text(configs, channel, actual): """ import functools from pathlib import Path -from typing import Union +from typing import Callable, Optional, Union from tested.configs import Bundle from tested.dodona import Status -from tested.evaluators.common import Evaluator, _curry_evaluator +from tested.evaluators.common import Evaluator, RawEvaluator, _curry_evaluator from tested.testsuite import ( EmptyChannel, ExceptionBuiltin, @@ -66,13 +66,16 @@ def get_evaluator( value, ) - currier = functools.partial(_curry_evaluator, bundle, context_dir) + currier: Callable[[RawEvaluator, Optional[dict]], Evaluator] = functools.partial( + _curry_evaluator, bundle, context_dir + ) # Handle channel states. if output == EmptyChannel.NONE: - return currier( - functools.partial(nothing.evaluate, unexpected_status=unexpected_status) + evaluator = functools.partial( + nothing.evaluate, unexpected_status=unexpected_status ) + return currier(evaluator) if output == IgnoredChannel.IGNORED: return currier(ignored.evaluate) if isinstance(output, ExitCodeOutputChannel): diff --git a/tested/evaluators/common.py b/tested/evaluators/common.py index 70dbf6b5..895e66fb 100644 --- a/tested/evaluators/common.py +++ b/tested/evaluators/common.py @@ -25,7 +25,7 @@ def evaluate_text(configs, channel, actual): import functools from dataclasses import field from pathlib import Path -from typing import Any, Callable, Dict, List, NamedTuple, Optional, Tuple, Union +from typing import Any, Callable, Dict, List, NamedTuple, Optional, Tuple from pydantic.dataclasses import dataclass @@ -92,23 +92,13 @@ def try_outputs( return actual, None -def get_status(status: Optional[Union[bool, Status]]) -> Status: - if status is None: - return Status.WRONG - elif isinstance(status, bool): - return Status.CORRECT if status else Status.WRONG - else: - return status - - def cleanup_specific_programmed( config: EvaluatorConfig, channel: NormalOutputChannel, actual: EvalResult ) -> EvalResult: - actual.result = get_status(actual.result) if isinstance(channel, ExceptionOutputChannel): lang_config = config.bundle.lang_config actual.readable_expected = lang_config.cleanup_stacktrace( - actual.readable_expected + actual.readable_expected or "" ) message = convert_stacktrace_to_clickable_feedback( lang_config, actual.readable_actual diff --git a/tested/evaluators/exception.py b/tested/evaluators/exception.py index 308576d0..ae1348b7 100644 --- a/tested/evaluators/exception.py +++ b/tested/evaluators/exception.py @@ -11,7 +11,7 @@ from tested.internationalization import get_i18n_string from tested.languages.utils import convert_stacktrace_to_clickable_feedback from tested.serialisation import ExceptionValue -from tested.testsuite import ExceptionOutputChannel +from tested.testsuite import ExceptionOutputChannel, OutputChannel logger = logging.getLogger(__name__) @@ -45,14 +45,14 @@ def try_as_readable_exception( def evaluate( - config: EvaluatorConfig, channel: ExceptionOutputChannel, actual: str + config: EvaluatorConfig, channel: OutputChannel, actual_str: str ) -> EvaluationResult: """ Evaluate an exception. :param config: Not used. :param channel: The channel from the test suite. - :param actual: The raw actual value of the execution. + :param actual_str: The raw actual value of the execution. :return: An evaluation result. """ @@ -63,7 +63,7 @@ def evaluate( language = config.bundle.global_config.dodona.programming_language readable_expected = expected.readable(language) - if not actual: + if not actual_str: return EvaluationResult( result=StatusMessage(enum=Status.WRONG), readable_expected=readable_expected, @@ -72,11 +72,11 @@ def evaluate( ) try: - actual = try_as_exception(config, actual) + actual = try_as_exception(config, actual_str) except (TypeError, ValueError) as e: staff_message = ExtendedMessage( description=get_i18n_string( - "evaluators.exception.staff", actual=actual, exception=e + "evaluators.exception.staff", actual=actual_str, exception=e ), format="text", permission=Permission.STAFF, diff --git a/tested/evaluators/exitcode.py b/tested/evaluators/exitcode.py index c5be1918..65ed5d97 100644 --- a/tested/evaluators/exitcode.py +++ b/tested/evaluators/exitcode.py @@ -4,7 +4,7 @@ from tested.dodona import Status, StatusMessage from tested.evaluators.common import EvaluationResult, EvaluatorConfig from tested.internationalization import get_i18n_string -from tested.testsuite import ExitCodeOutputChannel +from tested.testsuite import ExitCodeOutputChannel, OutputChannel logger = logging.getLogger(__name__) @@ -17,7 +17,7 @@ def _as_int(value: str) -> Optional[int]: def evaluate( - _config: EvaluatorConfig, channel: ExitCodeOutputChannel, value: str + _config: EvaluatorConfig, channel: OutputChannel, value: str ) -> EvaluationResult: assert isinstance(channel, ExitCodeOutputChannel) exit_code = _as_int(value) diff --git a/tested/evaluators/ignored.py b/tested/evaluators/ignored.py index 4eecbef6..0ce35a59 100644 --- a/tested/evaluators/ignored.py +++ b/tested/evaluators/ignored.py @@ -7,11 +7,11 @@ from tested.evaluators.common import EvaluationResult, EvaluatorConfig, try_outputs from tested.evaluators.exception import try_as_readable_exception from tested.evaluators.value import try_as_readable_value -from tested.testsuite import IgnoredChannel +from tested.testsuite import IgnoredChannel, OutputChannel def evaluate( - config: EvaluatorConfig, channel: IgnoredChannel, actual: str + config: EvaluatorConfig, channel: OutputChannel, actual: str ) -> EvaluationResult: assert isinstance(channel, IgnoredChannel) diff --git a/tested/evaluators/nothing.py b/tested/evaluators/nothing.py index ae5c437b..8545d132 100644 --- a/tested/evaluators/nothing.py +++ b/tested/evaluators/nothing.py @@ -8,12 +8,12 @@ from tested.evaluators.exception import try_as_readable_exception from tested.evaluators.value import try_as_readable_value from tested.internationalization import get_i18n_string -from tested.testsuite import EmptyChannel +from tested.testsuite import EmptyChannel, OutputChannel def evaluate( config: EvaluatorConfig, - channel: EmptyChannel, + channel: OutputChannel, actual: str, unexpected_status: Status = Status.WRONG, ) -> EvaluationResult: diff --git a/tested/evaluators/programmed.py b/tested/evaluators/programmed.py index d7e4643c..119fac7e 100644 --- a/tested/evaluators/programmed.py +++ b/tested/evaluators/programmed.py @@ -3,10 +3,10 @@ """ import logging import traceback -from typing import Optional +from typing import List, Optional -from tested.datatypes import StringTypes -from tested.dodona import ExtendedMessage, Permission, Status, StatusMessage +from tested.datatypes import BasicStringTypes +from tested.dodona import ExtendedMessage, Message, Permission, Status, StatusMessage from tested.evaluators.common import ( EvaluationResult, EvaluatorConfig, @@ -16,9 +16,8 @@ from tested.internationalization import get_i18n_string from tested.judge.programmed import evaluate_programmed from tested.judge.utils import BaseExecutionResult -from tested.serialisation import EvalResult, StringType, Value -from tested.testsuite import NormalOutputChannel, ProgrammedEvaluator -from tested.utils import get_args +from tested.serialisation import BooleanEvalResult, EvalResult, StringType, Value +from tested.testsuite import EvaluatorOutputChannel, OutputChannel, ProgrammedEvaluator _logger = logging.getLogger(__name__) @@ -27,7 +26,7 @@ def _maybe_string(value_: str) -> Optional[Value]: if value_: - return StringType(StringTypes.TEXT, value_) + return StringType(type=BasicStringTypes.TEXT, data=value_) else: return None @@ -37,23 +36,22 @@ def _try_specific(value_: str) -> EvalResult: def evaluate( - config: EvaluatorConfig, channel: NormalOutputChannel, actual: str + config: EvaluatorConfig, channel: OutputChannel, actual_str: str ) -> EvaluationResult: """ Evaluate using a programmed evaluator. This evaluator is unique, in that it is also responsible for running the evaluator (all other evaluators don't do that). """ - assert isinstance(channel, get_args(NormalOutputChannel)) - assert hasattr(channel, "evaluator") + assert isinstance(channel, EvaluatorOutputChannel) assert isinstance(channel.evaluator, ProgrammedEvaluator) - _logger.debug(f"Programmed evaluator for output {actual}") + _logger.debug(f"Programmed evaluator for output {actual_str}") # Convert the expected item to a Value, which is then passed to the # evaluator for evaluation. # This is slightly tricky, since the actual value must also be converted # to a value, and we are not yet sure what the actual value is exactly - result = get_values(config.bundle, channel, actual or "") + result = get_values(config.bundle, channel, actual_str or "") # TODO: why is this? if isinstance(result, EvaluationResult): return result @@ -103,9 +101,11 @@ def evaluate( messages=[stdout, stderr, DEFAULT_STUDENT], ) try: - evaluation_result = _try_specific(result.stdout) + evaluation_result = BooleanEvalResult.parse_raw( + result.stdout + ).as_eval_result() except (TypeError, ValueError): - messages = [ + messages: List[Message] = [ ExtendedMessage(description=DEFAULT_STUDENT, format="text"), ExtendedMessage( description=get_i18n_string("evaluators.programmed.result"), @@ -158,20 +158,20 @@ def evaluate( result_status = StatusMessage( enum=Status.CORRECT if evaluation_result.result else Status.WRONG ) - actual = cleanup_specific_programmed( + cleaned = cleanup_specific_programmed( config, channel, EvalResult( result=result_status.enum, - readable_expected=readable_expected, - readable_actual=readable_actual, + readable_expected=readable_expected, # type: ignore + readable_actual=readable_actual, # type: ignore messages=evaluation_result.messages, ), ) return EvaluationResult( result=result_status, - readable_expected=actual.readable_expected, - readable_actual=actual.readable_actual, - messages=actual.messages, + readable_expected=cleaned.readable_expected or "", + readable_actual=cleaned.readable_actual or "", + messages=cleaned.messages, ) diff --git a/tested/evaluators/specific.py b/tested/evaluators/specific.py index a3522a70..9495f909 100644 --- a/tested/evaluators/specific.py +++ b/tested/evaluators/specific.py @@ -1,5 +1,5 @@ """ -Specific evaluator +Evaluate the result of a language-specific oracle. """ from tested.dodona import ExtendedMessage, Permission, Status, StatusMessage @@ -9,20 +9,21 @@ cleanup_specific_programmed, ) from tested.internationalization import get_i18n_string -from tested.serialisation import EvalResult -from tested.testsuite import OutputChannel, SpecificEvaluator +from tested.serialisation import BooleanEvalResult +from tested.testsuite import EvaluatorOutputChannel, OutputChannel, SpecificEvaluator def evaluate( - config: EvaluatorConfig, channel: OutputChannel, actual: str + config: EvaluatorConfig, channel: OutputChannel, actual_str: str ) -> EvaluationResult: """ Compare the result of a specific evaluator. This evaluator has no options. """ + assert isinstance(channel, EvaluatorOutputChannel) assert isinstance(channel.evaluator, SpecificEvaluator) # Special support for no values to have a better error message. - if actual == "": + if actual_str == "": return EvaluationResult( result=StatusMessage( enum=Status.WRONG, @@ -33,13 +34,12 @@ def evaluate( messages=[get_i18n_string("evaluators.specific.missing.message")], ) - # Try parsing as the result. try: - actual: EvalResult = EvalResult.parse_raw(actual) + actual = BooleanEvalResult.parse_raw(actual_str).as_eval_result() except (TypeError, ValueError) as e: staff_message = ExtendedMessage( description=get_i18n_string( - "evaluators.specific.staff", actual=actual, e=e + "evaluators.specific.staff", actual=actual_str, e=e ), format="text", permission=Permission.STAFF, @@ -59,7 +59,7 @@ def evaluate( return EvaluationResult( result=StatusMessage(enum=actual.result), - readable_expected=actual.readable_expected, - readable_actual=actual.readable_actual, + readable_expected=actual.readable_expected or "", + readable_actual=actual.readable_actual or "", messages=actual.messages, ) diff --git a/tested/evaluators/text.py b/tested/evaluators/text.py index f3512943..61527b15 100644 --- a/tested/evaluators/text.py +++ b/tested/evaluators/text.py @@ -101,7 +101,7 @@ def evaluate_text( def evaluate_file( - config: EvaluatorConfig, channel: FileOutputChannel, actual: str + config: EvaluatorConfig, channel: OutputChannel, actual: str ) -> EvaluationResult: """ Evaluate the contents of two files. The file evaluator supports one option, diff --git a/tested/evaluators/value.py b/tested/evaluators/value.py index 26a2ff7f..a1425c7d 100644 --- a/tested/evaluators/value.py +++ b/tested/evaluators/value.py @@ -1,8 +1,9 @@ """ Value evaluator. """ +import itertools import logging -from typing import Optional, Tuple, Union +from typing import Optional, Tuple, Union, cast from tested.configs import Bundle from tested.datatypes import ( @@ -27,8 +28,13 @@ parse_value, to_python_comparable, ) -from tested.testsuite import OutputChannel, TextOutputChannel, ValueOutputChannel -from tested.utils import get_args, sorted_no_duplicates +from tested.testsuite import ( + EvaluatorOutputChannel, + OutputChannel, + TextOutputChannel, + ValueOutputChannel, +) +from tested.utils import sorted_no_duplicates logger = logging.getLogger(__name__) @@ -45,29 +51,30 @@ def try_as_readable_value( def get_values( - bundle: Bundle, output_channel: ValueOutputChannel, actual: str + bundle: Bundle, output_channel: EvaluatorOutputChannel, actual_str: str ) -> Union[EvaluationResult, Tuple[Value, str, Optional[Value], str]]: if isinstance(output_channel, TextOutputChannel): expected = output_channel.get_data_as_string(bundle.config.resources) expected_value = StringType(type=BasicStringTypes.TEXT, data=expected) - actual_value = StringType(type=BasicStringTypes.TEXT, data=actual) - return expected_value, expected, actual_value, actual + actual_value = StringType(type=BasicStringTypes.TEXT, data=actual_str) + return expected_value, expected, actual_value, actual_str assert isinstance(output_channel, ValueOutputChannel) expected = output_channel.value + assert isinstance(expected, Value) readable_expected = generate_statement(bundle, expected) # Special support for empty strings. - if not actual.strip(): + if not actual_str.strip(): return expected, readable_expected, None, "" # A crash here indicates a problem with one of the language implementations, # or a student is trying to cheat. try: - actual = parse_value(actual) + actual = parse_value(actual_str) except (TypeError, ValueError) as e: - raw_message = f"Received {actual}, which caused {e} for get_values." + raw_message = f"Received {actual_str}, which caused {e} for get_values." message = ExtendedMessage( description=raw_message, format="text", permission=Permission.STAFF ) @@ -75,7 +82,7 @@ def get_values( return EvaluationResult( result=StatusMessage(enum=Status.INTERNAL_ERROR, human=student), readable_expected=readable_expected, - readable_actual=str(actual), + readable_actual=actual_str, messages=[message], ) @@ -83,57 +90,37 @@ def get_values( return expected, readable_expected, actual, readable_actual -def _check_type(bundle: Bundle, expected: Value, actual: Value) -> Tuple[bool, Value]: - valid, value = check_data_type(bundle, expected, actual) - if not valid: - return False, value - elif isinstance(value.type, get_args(SimpleTypes)): - return True, value - elif isinstance(value, SequenceType): - if as_basic_type(value).type == BasicSequenceTypes.SET: - actual_object = sorted_no_duplicates( - value.data, recursive_key=lambda x: x.data - ) - expected_object = sorted_no_duplicates( - expected.data, recursive_key=lambda x: x.data - ) - else: - actual_object, expected_object = value.data, expected.data - data = [] - for actual_element, expected_element in zip(actual_object, expected_object): - element_valid, element_value = _check_type( - bundle, expected_element, actual_element +def _prepare_value_for_type_check(value: Value) -> Value: + """ + Prepare a value for type checking. + + This is mainly sorting the values if the value is a container. Note that we don't + actually do type checking here. + + :param value: The value to prepare. + :return: The prepared value. + """ + if isinstance(value, SequenceType): + basic_type = as_basic_type(value) + if basic_type.type == BasicSequenceTypes.SET: + value.data = sorted_no_duplicates( + value.data, recursive_key=lambda x: x.data # type: ignore ) - valid = valid and element_valid - data.append(element_value) - value.data = data - return valid, value - else: + elif isinstance(value, ObjectType): assert isinstance(value, ObjectType) - data = [] - actual_object = sorted_no_duplicates( - value.data, key=lambda x: x.key, recursive_key=lambda x: x.data - ) - expected_object = sorted_no_duplicates( - expected.data, key=lambda x: x.key, recursive_key=lambda x: x.data + value.data = sorted_no_duplicates( + value.data, key=lambda x: x.key, recursive_key=lambda x: x.data # type: ignore ) - - for actual_element, expected_element in zip(actual_object, expected_object): - actual_key, actual_value = actual_element.key, actual_element.value - expected_key, expected_value = actual_element.key, actual_element.value - key_valid, key_value = _check_type(bundle, expected_key, actual_key) - value_valid, value_value = _check_type(bundle, expected_value, actual_value) - valid = valid and key_valid and value_valid - data.append(ObjectKeyValuePair(key=key_value, value=value_value)) - value.data = data - return valid, value + else: + assert isinstance(value.type, SimpleTypes) + return value -def check_data_type( +def _check_simple_type( bundle: Bundle, expected: Value, actual: Value ) -> Tuple[bool, Value]: """ - Check if the type of the two values match. The following procedure is used: + Check if the data type of two simple values match. The following procedure is used: 1. If the expected value's type is a basic type, the actual value is reduced to it's basic type, after which both types are checked for equality. @@ -148,11 +135,10 @@ def check_data_type( type. :param bundle: The configuration bundle. - :param expected: The expected type from the test suite. - :param actual: The actual type produced by the suite. + :param expected: The expected value from the test suite. + :param actual: The actual value produced by the suite. - :return: A tuple with the result and expected value, the type that was used to - do the check. + :return: A tuple with the result and expected value, which can have a modified type. """ supported_types = fallback_type_support_map(bundle.lang_config) @@ -161,11 +147,11 @@ def check_data_type( raise ValueError(f"The language does not support {expected.type}") # Case 1. - if isinstance(expected.type, get_args(BasicTypes)): + if isinstance(expected.type, BasicTypes): basic_actual = as_basic_type(actual) return expected.type == basic_actual.type, expected - assert isinstance(expected.type, get_args(AdvancedTypes)) + assert isinstance(expected.type, AdvancedTypes) # Case 2.b. if supported_types[expected.type] == TypeSupport.REDUCED: @@ -179,8 +165,80 @@ def check_data_type( return expected.type == actual.type, expected +def _check_data_type( + bundle: Bundle, expected: Value, actual: Optional[Value] +) -> Tuple[bool, Value]: + """ + Check if two values have the same (recursive) type. + + Recursive in this context means that all container types need to have elements + with matching types as well. For example, a list of integers will only match + another list of integers. + + :param bundle: The configuration bundle. + :param expected: The expected value from the test suite. + :param actual: The actual value produced by the suite. + + :return: A tuple with the result and expected value, which can have a modified type. + """ + prepared_expected = _prepare_value_for_type_check(expected) + + if actual is None: + return False, prepared_expected + + prepared_actual = _prepare_value_for_type_check(actual) + + valid, prepared_expected = _check_simple_type( + bundle, prepared_expected, prepared_actual + ) + + if isinstance(prepared_expected, SequenceType): + expected_elements = prepared_expected.data + actual_elements = ( + prepared_actual.data if isinstance(prepared_actual.data, list) else [] + ) + prepared_elements = [] + for expected_element, actual_element in itertools.zip_longest( + expected_elements, actual_elements + ): + assert expected_element is None or isinstance(expected_element, Value) + assert actual_element is None or isinstance(actual_element, Value) + element_valid, prepared_element = _check_data_type( + bundle, expected_element, actual_element + ) + prepared_elements.append(prepared_element) + valid = valid and element_valid + prepared_expected.data = prepared_elements + elif isinstance(prepared_expected, ObjectType): + expected_elements = prepared_expected.data + actual_elements = ( + prepared_actual.data if isinstance(prepared_actual.data, list) else [] + ) + prepared_elements = [] + for expected_element, actual_element in itertools.zip_longest( + expected_elements, actual_elements + ): + assert isinstance(actual_element, ObjectKeyValuePair) + actual_key, actual_value = actual_element.key, actual_element.value + assert isinstance(actual_key, Value) and isinstance(actual_value, Value) + expected_key, expected_value = actual_element.key, actual_element.value + assert isinstance(expected_key, Value) and isinstance(expected_value, Value) + key_valid, prepared_key = _check_data_type(bundle, expected_key, actual_key) + value_valid, prepared_value = _check_data_type( + bundle, expected_value, actual_value + ) + valid = valid and key_valid and value_valid + prepared_elements.append( + ObjectKeyValuePair(key=prepared_key, value=prepared_value) + ) + prepared_expected.data = prepared_elements + else: + assert isinstance(prepared_expected.type, SimpleTypes) + return valid, prepared_expected + + def evaluate( - config: EvaluatorConfig, channel: OutputChannel, actual: str + config: EvaluatorConfig, channel: OutputChannel, actual_str: str ) -> EvaluationResult: """ Evaluate two values. The values must match exact. Currently, this evaluator @@ -192,14 +250,14 @@ def evaluate( # Try parsing the value as an EvaluationResult. # This is the result of a custom evaluator. try: - evaluation_result = EvaluationResult.__pydantic_model__.parse_raw(actual) + evaluation_result = EvaluationResult.__pydantic_model__.parse_raw(actual_str) # type: ignore except (TypeError, ValueError): pass else: return evaluation_result # Try parsing the value as an actual Value. - result = get_values(config.bundle, channel, actual) + result = get_values(config.bundle, channel, actual_str) if isinstance(result, EvaluationResult): return result else: @@ -209,7 +267,7 @@ def evaluate( is_multiline_string = ( config.options.get("stringsAsText", True) and expected.type == BasicStringTypes.TEXT - and "\n" in expected.data + and "\n" in cast(str, expected.data) ) if is_multiline_string: readable_expected = get_as_string(expected, readable_expected) @@ -224,7 +282,7 @@ def evaluate( readable_actual=readable_actual, ) - type_check, expected = _check_type(config.bundle, expected, actual) + type_check, expected = _check_data_type(config.bundle, expected, actual) messages = [] type_status = None @@ -259,13 +317,13 @@ def evaluate( ) -def get_as_string(value: Optional[Value], readable: str): +def get_as_string(value: Optional[Value], readable: str) -> str: # Return readable if value is none if value is None: return readable - # Replace tab by 4 spaces - return ( - value.data.replace("\t", " ") - if value.type == BasicStringTypes.TEXT - else readable - ) + if value.type == BasicStringTypes.TEXT: + assert isinstance(value, StringType) + # Replace tab by 4 spaces + return value.data.replace("\t", " ") + else: + return readable diff --git a/tested/features.py b/tested/features.py index b5268e5c..0356cce2 100644 --- a/tested/features.py +++ b/tested/features.py @@ -4,12 +4,11 @@ import logging import operator from collections import defaultdict -from enum import Enum, StrEnum, auto, unique +from enum import StrEnum, auto, unique from functools import reduce from typing import TYPE_CHECKING, Dict, Iterable, NamedTuple, Set from tested.datatypes import AllTypes, BasicObjectTypes, BasicSequenceTypes, NestedTypes -from tested.utils import fallback if TYPE_CHECKING: from tested.languages.config import Language @@ -59,7 +58,7 @@ def get_used_features(self) -> FeatureSet: NOTHING = FeatureSet(constructs=set(), types=set(), nested_types=set()) -class TypeSupport(Enum): +class TypeSupport(StrEnum): SUPPORTED = auto() """ The type is fully supported. @@ -97,13 +96,13 @@ def fallback_type_support_map(language: "Language") -> Dict[AllTypes, TypeSuppor :return: The typing support dict. """ - config = dict() + config = defaultdict(lambda: TypeSupport.UNSUPPORTED) for x, y in language.datatype_support().items(): if isinstance(y, str): config[x] = TypeSupport[y.upper()] else: config[x] = y - return fallback(defaultdict(lambda: TypeSupport.UNSUPPORTED), config) + return config def combine_features(iterable: Iterable[FeatureSet]) -> FeatureSet: @@ -130,6 +129,7 @@ def is_supported(language: "Language") -> bool: :return: True or False """ + assert language.config is not None required = language.config.suite.get_used_features() # Check constructs @@ -150,6 +150,7 @@ def is_supported(language: "Language") -> bool: # Check language-specific evaluators for tab in language.config.suite.tabs: + assert tab.contexts is not None for context in tab.contexts: for testcase in context.testcases: languages = testcase.output.get_specific_eval_languages() @@ -160,15 +161,16 @@ def is_supported(language: "Language") -> bool: f"{languages}!" ) return False + nested_types = [] + for key, value_types in required.nested_types: + if key in (BasicSequenceTypes.SET, BasicObjectTypes.MAP): + nested_types.append((key, value_types)) - nested_types = filter( - lambda x: x[0] in (BasicSequenceTypes.SET, BasicObjectTypes.MAP), - required.nested_types, - ) restricted = { BasicSequenceTypes.SET: language.set_type_restrictions(), BasicObjectTypes.MAP: language.map_type_restrictions(), } + for key, value_types in nested_types: if not (value_types <= restricted[key]): _logger.warning("This test suite is not compatible!") diff --git a/tested/instantiate_exercise.py b/tested/instantiate_exercise.py index 0be63871..4fd765cb 100644 --- a/tested/instantiate_exercise.py +++ b/tested/instantiate_exercise.py @@ -18,7 +18,7 @@ ) from tested.dsl import parse_dsl from tested.features import fallback_type_support_map -from tested.languages import LANGUAGES, Language, get_language, language_exists +from tested.languages import LANGUAGES, get_language, language_exists from tested.testsuite import Suite, parse_test_suite @@ -139,8 +139,8 @@ def _filter_valid_languages(languages: List[str], test_suite: Suite) -> List[str :return: all given languages which support the test suite """ - def is_supported(language: str) -> bool: - language: Language = get_language(None, language) + def is_supported(language_str: str) -> bool: + language = get_language(None, language_str) from tested.features import TypeSupport @@ -159,9 +159,9 @@ def is_supported(language: str) -> bool: # Check language-specific evaluators for testcase in ( testcase - for tab in test_suite.tabs - for context in tab.contexts - for testcase in context.all_testcases() + for tab in test_suite.tabs # type: ignore + for context in tab.contexts # type: ignore + for testcase in context.testcases ): eval_langs = testcase.output.get_specific_eval_languages() if eval_langs is not None and language not in eval_langs: @@ -176,7 +176,7 @@ def is_supported(language: str) -> bool: BasicObjectTypes.MAP: language.map_type_restrictions(), } for key, value_types in nested_types: - if not (value_types <= restricted[key]): + if not (value_types <= restricted[key]): # type: ignore return False return True @@ -301,7 +301,7 @@ def _instantiate_descriptions( if description.is_template: # Generate instance = create_description_instance_from_template( - description.template, + description.template, # type: ignore language, description.natural_language, test_suite.namespace, diff --git a/tested/judge/collector.py b/tested/judge/collector.py index 563637fc..12fea125 100644 --- a/tested/judge/collector.py +++ b/tested/judge/collector.py @@ -15,6 +15,7 @@ CloseContext, CloseJudgment, CloseTab, + CloseTest, CloseTestcase, EscalateStatus, ExtendedMessage, @@ -257,11 +258,11 @@ def terminate(self, status: Union[Status, StatusMessage]): # Add stack. for to_close in reversed(self.tree_stack): - try: - # noinspection PyArgumentList - command = close_for(to_close)(status=status) - except TypeError: - command = close_for(to_close)() + closer = close_for(to_close) + if closer == CloseTest: + command = CloseTest(generated="", status=status) + else: + command = closer() # type: ignore self._add(command) self.collected = True diff --git a/tested/judge/compilation.py b/tested/judge/compilation.py index 63203df1..48365296 100644 --- a/tested/judge/compilation.py +++ b/tested/judge/compilation.py @@ -16,7 +16,7 @@ def run_compilation( - bundle: Bundle, directory: Path, dependencies: List[Path], remaining: float + bundle: Bundle, directory: Path, dependencies: List[str], remaining: float ) -> Tuple[Optional[BaseExecutionResult], Union[List[str], FileFilter]]: """ The compilation step in the pipeline. This callback is used in both the @@ -53,7 +53,7 @@ def run_compilation( decide to fallback to individual mode if the compilation result is not positive. """ - command, files = bundle.lang_config.compilation([str(x) for x in dependencies]) + command, files = bundle.lang_config.compilation(dependencies) _logger.debug( "Generating files with command %s in directory %s", command, directory ) diff --git a/tested/judge/core.py b/tested/judge/core.py index a3527e65..b3e52646 100644 --- a/tested/judge/core.py +++ b/tested/judge/core.py @@ -1,4 +1,3 @@ -import concurrent import logging import shutil import time @@ -117,7 +116,7 @@ def judge(bundle: Bundle): index = len(bundle.suite.tabs) + 1 if messages: collector.prepare_tab( - StartTab(get_i18n_string("judge.core.compilation")), index + StartTab(title=get_i18n_string("judge.core.compilation")), index ) for message in messages: collector.add(AppendMessage(message=message)) @@ -143,7 +142,7 @@ def judge(bundle: Bundle): _logger.info("Compilation error, falling back to individual mode") # Remove the selector file from the dependencies. # Otherwise, it will keep being compiled, which we want to avoid. - if bundle.lang_config.needs_selector(): + if selector and bundle.lang_config.needs_selector(): files.remove(selector) # When compilation succeeded, only add annotations elif status == Status.CORRECT: @@ -155,7 +154,7 @@ def judge(bundle: Bundle): # Report messages. if messages: collector.add_tab( - StartTab(get_i18n_string("judge.core.compilation")), -1 + StartTab(title=get_i18n_string("judge.core.compilation")), -1 ) for message in messages: collector.add(AppendMessage(message=message)) @@ -181,6 +180,7 @@ def judge(bundle: Bundle): # Create a list of runs we want to execute. for tab_index, tab in enumerate(bundle.suite.tabs): collector.add_tab(StartTab(title=tab.name, hidden=tab.hidden), tab_index) + assert tab.contexts execution_units = merge_contexts_into_units(tab.contexts) executions = [] offset = 0 @@ -287,7 +287,7 @@ def evaluation_function(_eval_remainder): # Ensure finally is called NOW and cancels remaining tasks. del results return status - except concurrent.futures.TimeoutError: + except TimeoutError: _logger.warning("Futures did not end soon enough.", exc_info=True) return Status.TIME_LIMIT_EXCEEDED @@ -323,6 +323,7 @@ def _generate_files( execution_names = [] # Generate the files for each execution. for tab_i, tab in enumerate(bundle.suite.tabs): + assert tab.contexts execution_units = merge_contexts_into_units(tab.contexts) for unit_i, unit in enumerate(execution_units): exec_name = execution_name(bundle.lang_config, tab_i, unit_i) diff --git a/tested/judge/evaluation.py b/tested/judge/evaluation.py index e2db2281..74173923 100644 --- a/tested/judge/evaluation.py +++ b/tested/judge/evaluation.py @@ -1,8 +1,9 @@ import html import logging +from collections.abc import Collection from enum import StrEnum, unique from pathlib import Path -from typing import Iterable, List, Optional, Set, Tuple +from typing import List, Optional, Set, Tuple from tested.configs import Bundle from tested.dodona import ( @@ -49,7 +50,7 @@ ValueOutput, ValueOutputChannel, ) -from tested.utils import get_args, safe_del, safe_get +from tested.utils import safe_del, safe_get _logger = logging.getLogger(__name__) @@ -117,7 +118,7 @@ def _evaluate_channel( expected: str if ( - not isinstance(output, get_args(SpecialOutputChannel)) + not isinstance(output, SpecialOutputChannel) and not output.show_expected and not is_correct ): @@ -220,7 +221,7 @@ def evaluate_context_results( if not could_delete: _logger.warning("Missing output in context testcase.") missing_values.append( - AppendMessage(get_i18n_string("judge.evaluation.early-exit")) + AppendMessage(message=get_i18n_string("judge.evaluation.early-exit")) ) missing_values.append( EscalateStatus( @@ -234,7 +235,7 @@ def evaluate_context_results( if recovered := "\n".join(stdout_): missing_values.append( AppendMessage( - ExtendedMessage( + message=ExtendedMessage( description="Standaarduitvoer was:\n" + recovered, format="code" ) ) @@ -242,7 +243,7 @@ def evaluate_context_results( if recovered := "\n".join(stderr_): missing_values.append( AppendMessage( - ExtendedMessage( + message=ExtendedMessage( description="Standaardfout was:\n" + recovered, format="code" ) ) @@ -362,7 +363,7 @@ def evaluate_context_results( def _link_files_message( - link_files: Iterable[FileUrl], collector: Optional[OutputManager] = None + link_files: Collection[FileUrl], collector: Optional[OutputManager] = None ) -> Optional[AppendMessage]: link_list = ", ".join( f'' @@ -395,23 +396,23 @@ def should_show(test: OutputChannel, channel: Channel) -> bool: if channel == Channel.EXIT: if test == IgnoredChannel.IGNORED: return False - assert isinstance(test, get_args(ExitCodeOutputChannel)) + assert isinstance(test, ExitCodeOutputChannel) return test.value != 0 elif channel in (Channel.STDOUT, Channel.STDERR): - assert isinstance(test, get_args(TextOutput)) + assert isinstance(test, TextOutput) # We don't show the channel if the output is nothing or ignored. - return not isinstance(test, get_args(SpecialOutputChannel)) + return not isinstance(test, SpecialOutputChannel) elif channel == Channel.FILE: - assert isinstance(test, get_args(FileOutput)) + assert isinstance(test, FileOutput) # We don't show the channel if we ignore the channel. return not isinstance(test, IgnoredChannel) elif channel == Channel.RETURN: - assert isinstance(test, get_args(ValueOutput)) + assert isinstance(test, ValueOutput) # We don't show the channel if we ignore it or expect no result. - return not isinstance(test, get_args(SpecialOutputChannel)) + return not isinstance(test, SpecialOutputChannel) elif channel == Channel.EXCEPTION: - assert isinstance(test, get_args(ExceptionOutput)) - return not isinstance(test, get_args(SpecialOutputChannel)) + assert isinstance(test, ExceptionOutput) + return not isinstance(test, SpecialOutputChannel) else: raise AssertionError(f"Unknown channel {channel}") @@ -427,7 +428,7 @@ def guess_expected_value(bundle: Bundle, test: OutputChannel) -> str: :return: A best effort attempt of the expected value. """ - if isinstance(test, get_args(SpecialOutputChannel)): + if isinstance(test, SpecialOutputChannel): return "" elif isinstance(test, TextOutputChannel): return test.get_data_as_string(bundle.config.resources) @@ -447,7 +448,7 @@ def guess_expected_value(bundle: Bundle, test: OutputChannel) -> str: ) elif isinstance(test, ExitCodeOutputChannel): return str(test.value) - _logger.warn(f"Unknown output type {test}") + _logger.warning(f"Unknown output type {test}") return "" @@ -484,6 +485,7 @@ def prepare_evaluation(bundle: Bundle, collector: OutputManager): for i, tab in enumerate(bundle.suite.tabs): collector.prepare_tab(StartTab(title=tab.name, hidden=tab.hidden), i) + assert tab.contexts for j, context in enumerate(tab.contexts): updates = [] diff --git a/tested/judge/execution.py b/tested/judge/execution.py index d31ff3b1..fa7673cf 100644 --- a/tested/judge/execution.py +++ b/tested/judge/execution.py @@ -4,7 +4,7 @@ import time from dataclasses import dataclass from pathlib import Path -from typing import Callable, List, Optional, Tuple, Union +from typing import List, Optional, Tuple, Union, cast from tested.configs import Bundle from tested.dodona import Message, Status @@ -14,7 +14,7 @@ from tested.languages.config import FileFilter from tested.languages.conventionalize import EXECUTION_PREFIX, selector_name from tested.languages.preparation import exception_file, value_file -from tested.testsuite import Context, EmptyChannel, ExecutionMode +from tested.testsuite import Context, EmptyChannel, ExecutionMode, MainInput from tested.utils import safe_del _logger = logging.getLogger(__name__) @@ -131,7 +131,7 @@ class Execution: execution_index: int mode: ExecutionMode common_directory: Path - files: Union[List[str], Callable[[Path, str], bool]] + files: Union[List[str], FileFilter] precompilation_result: Optional[Tuple[List[Message], Status]] collector: OutputManager @@ -237,7 +237,8 @@ def execute_execution( if args.mode == ExecutionMode.INDIVIDUAL: _logger.info("Compiling context %s in INDIVIDUAL mode...", args.execution_name) remaining = max_time - (time.perf_counter() - start) - result, files = run_compilation(bundle, execution_dir, dependencies, remaining) + deps = [str(x) for x in dependencies] + result, files = run_compilation(bundle, execution_dir, deps, remaining) # A new compilation means a new file filtering files = filter_files(files, execution_dir) @@ -380,7 +381,7 @@ def merge_contexts_into_units(contexts: List[Context]) -> List[ExecutionUnit]: # If we get stdin, start a new execution unit. if ( context.has_main_testcase() - and context.testcases[0].input.stdin != EmptyChannel.NONE + and cast(MainInput, context.testcases[0].input).stdin != EmptyChannel.NONE ): if current_unit: units.append(ExecutionUnit(contexts=current_unit)) diff --git a/tested/judge/programmed.py b/tested/judge/programmed.py index 6331bd2f..7d7ea9d2 100644 --- a/tested/judge/programmed.py +++ b/tested/judge/programmed.py @@ -7,10 +7,11 @@ import sys import traceback import types +from collections.abc import Generator from dataclasses import dataclass, field from io import StringIO from pathlib import Path -from typing import ContextManager, List, Optional, Tuple, Union +from typing import List, Optional, Tuple, Union, cast from tested.configs import Bundle, create_bundle from tested.dodona import ExtendedMessage, Permission, Status @@ -23,7 +24,7 @@ generate_custom_evaluator, generate_statement, ) -from tested.serialisation import EvalResult, Value +from tested.serialisation import BooleanEvalResult, EvalResult, Value from tested.testsuite import ProgrammedEvaluator from tested.utils import get_identifier @@ -33,7 +34,7 @@ def evaluate_programmed( bundle: Bundle, evaluator: ProgrammedEvaluator, - expected: Optional[Value], + expected: Value, actual: Value, ) -> Union[BaseExecutionResult, EvalResult]: """ @@ -54,7 +55,7 @@ def evaluate_programmed( def _evaluate_others( bundle: Bundle, evaluator: ProgrammedEvaluator, - expected: Optional[Value], + expected: Value, actual: Value, ) -> BaseExecutionResult: """ @@ -165,7 +166,7 @@ def _evaluate_others( @contextlib.contextmanager -def _catch_output() -> ContextManager[Tuple[StringIO, StringIO]]: +def _catch_output() -> Generator[Tuple[StringIO, StringIO], None, None]: old_stdout = sys.stdout old_stderr = sys.stderr stdout = StringIO() @@ -190,7 +191,7 @@ class _EvaluationResult: def _evaluate_python( bundle: Bundle, evaluator: ProgrammedEvaluator, - expected: Optional[Value], + expected: Value, actual: Value, ) -> EvalResult: """ @@ -239,9 +240,6 @@ def _evaluate_python( stdout_ = stdout_.getvalue() stderr_ = stderr_.getvalue() - print(stdout_) - print(stderr_) - messages = [] if stdout_: messages.append( @@ -265,8 +263,7 @@ def _evaluate_python( ) ) - # noinspection PyTypeChecker - result_: Optional[_EvaluationResult] = global_env["__tested_test__result"] + result_ = cast(_EvaluationResult | None, global_env["__tested_test__result"]) # If the result is None, the evaluator is broken. if result_ is None: @@ -284,20 +281,20 @@ def _evaluate_python( ) return EvalResult( result=Status.INTERNAL_ERROR, - readable_expected=None, - readable_actual=None, + readable_expected=None, # type: ignore + readable_actual=None, # type: ignore messages=messages, ) assert isinstance(result_, _EvaluationResult) try: - return EvalResult( + return BooleanEvalResult( result=result_.result, - readable_expected=result_.readable_expected, - readable_actual=result_.readable_actual, + readable_expected=result_.readable_expected, # type: ignore + readable_actual=result_.readable_actual, # type: ignore messages=messages + result_.messages, - ) + ).as_eval_result() except (TypeError, ValueError): # This happens when the messages are not in the correct format. In normal # execution, this is caught when parsing the resulting json, but this does @@ -323,7 +320,7 @@ def _evaluate_python( ) return EvalResult( result=Status.INTERNAL_ERROR, - readable_expected=None, - readable_actual=None, + readable_expected=None, # type: ignore + readable_actual=None, # type: ignore messages=messages, ) diff --git a/tested/judge/utils.py b/tested/judge/utils.py index 83cfde40..d43da8d3 100644 --- a/tested/judge/utils.py +++ b/tested/judge/utils.py @@ -58,8 +58,8 @@ def run_command( ) except subprocess.TimeoutExpired as e: return BaseExecutionResult( - stdout=e.stdout or "", - stderr=e.stderr or "", + stdout=str(e.stdout or ""), + stderr=str(e.stderr or ""), exit=0, timeout=True, memory=False, diff --git a/tested/languages/bash/config.py b/tested/languages/bash/config.py index 35265ebc..be6f993a 100644 --- a/tested/languages/bash/config.py +++ b/tested/languages/bash/config.py @@ -75,6 +75,7 @@ def linter(self, remaining: float) -> Tuple[List[Message], List[AnnotateCode]]: # Import locally to prevent errors. from tested.languages.bash import linter + assert self.config return linter.run_shellcheck(self.config.dodona, remaining) def generate_statement(self, statement: Statement) -> str: diff --git a/tested/languages/bash/generators.py b/tested/languages/bash/generators.py index 70f29e29..2e478a0f 100644 --- a/tested/languages/bash/generators.py +++ b/tested/languages/bash/generators.py @@ -1,11 +1,12 @@ import shlex -from typing import List +from typing import Callable, List from tested.datatypes import AdvancedStringTypes, BasicStringTypes from tested.languages.preparation import ( PreparedContext, PreparedExecutionUnit, PreparedTestcase, + PreparedTestcaseStatement, ) from tested.languages.utils import convert_unknown_type from tested.serialisation import ( @@ -14,12 +15,14 @@ FunctionType, Identifier, Statement, + StringType, Value, ) -from tested.utils import get_args +from tested.testsuite import MainInput def convert_value(value: Value) -> str: + assert isinstance(value, StringType), f"Invalid literal: {value!r}" if value.type in (AdvancedStringTypes.CHAR, BasicStringTypes.TEXT): return shlex.quote(value.data) elif value.type == BasicStringTypes.UNKNOWN: @@ -31,9 +34,11 @@ def has_nested_function_call(fun: FunctionCall) -> bool: return len(list(filter(lambda a: isinstance(a, FunctionCall), fun.arguments))) > 0 -# TODO: can we merge these two functions? def convert_function_call_nested( - function: FunctionCall, index: int, index_fun: callable, index_map: list + function: FunctionCall, + index_fun: Callable[[], int], + index_map: list, + index=int, ) -> str: index_map.append({}) result = "" @@ -47,7 +52,7 @@ def convert_function_call_nested( if has_nested_function_call(argument): result += ( convert_function_call_nested( - argument, index_map[-1][i], index_fun, index_map + argument, index_fun, index_map, index_map[-1][i] ) + "\n" ) @@ -69,7 +74,7 @@ def convert_function_call_nested( ) else: result += f'"$ARG{index_map[-1][i]}" ' - elif isinstance(argument, get_args(Value)): + elif isinstance(argument, Value): # We have a value, delegate to the value template. result += convert_value(argument) + " " result += ")" @@ -78,7 +83,9 @@ def convert_function_call_nested( def convert_function_call( - function: FunctionCall, index_fun: callable, index_map: list + function: FunctionCall, + index_fun: Callable[[], int], + index_map: list, ) -> str: index_map.append({}) result = "" @@ -92,7 +99,7 @@ def convert_function_call( if has_nested_function_call(argument): result += ( convert_function_call_nested( - argument, index_map[-1][i], index_fun, index_map + argument, index_fun, index_map, index_map[-1][i] ) + "\n" ) @@ -118,7 +125,7 @@ def convert_function_call( ) else: result += f'"$ARG{index_map[-1][i]}" ' - elif isinstance(argument, get_args(Value)): + elif isinstance(argument, Value): # We have a value, delegate to the value template. result += convert_value(argument) + " " else: @@ -143,9 +150,9 @@ def convert_statement(statement: Statement) -> str: elif isinstance(statement, FunctionCall): index_fun = unique_index_function() return convert_function_call(statement, index_fun, []) - elif isinstance(statement, get_args(Value)): + elif isinstance(statement, Value): return convert_value(statement) - elif isinstance(statement, get_args(Assignment)): + elif isinstance(statement, Assignment): result = f"local {statement.variable}=" if isinstance(statement.expression, FunctionCall): result += f"$({convert_statement(statement.expression)})" @@ -202,9 +209,11 @@ def convert_execution_unit(pu: PreparedExecutionUnit) -> str: # Prepare command arguments if needed. if tc.testcase.is_main_testcase(): + assert isinstance(tc.input, MainInput) result += f"{indent}source {pu.submission_name}.sh " result += " ".join(shlex.quote(s) for s in tc.input.arguments) + "\n" else: + assert isinstance(tc.input, PreparedTestcaseStatement) result += indent + convert_statement(tc.input.input_statement()) + "\n" result += "\n" diff --git a/tested/languages/bash/linter.py b/tested/languages/bash/linter.py index a8541f1a..09e7c103 100644 --- a/tested/languages/bash/linter.py +++ b/tested/languages/bash/linter.py @@ -27,15 +27,23 @@ def run_shellcheck( """ submission = config.source language_options = config.config_for() - if language_options.get("shellcheck_config", None): - config_path = config.resources / language_options.get("shellcheck_config") + if path := language_options.get("shellcheck_config", None): + assert isinstance(path, str) + config_path = config.resources / path # Add shellcheck file in home folder - shutil.copy2(Path(config_path), Path(Path.home(), ".shellcheckrc")) + shutil.copy2(config_path, Path.home() / ".shellcheckrc") execution_results = run_command( directory=submission.parent, timeout=remaining, - command=["shellcheck", "-f", "json", "-s", language, submission.absolute()], + command=[ + "shellcheck", + "-f", + "json", + "-s", + language, + str(submission.absolute()), + ], ) if execution_results is None: diff --git a/tested/languages/c/config.py b/tested/languages/c/config.py index 655d3f0f..df185e01 100644 --- a/tested/languages/c/config.py +++ b/tested/languages/c/config.py @@ -44,7 +44,7 @@ def supported_constructs(self) -> Set[Construct]: } def datatype_support(self) -> Mapping[AllTypes, TypeSupport]: - return { + return { # type: ignore "integer": "supported", "real": "supported", "char": "supported", @@ -70,6 +70,7 @@ def compilation(self, files: List[str]) -> CallbackResult: main_file = files[-1] exec_file = Path(main_file).stem result = executable_name(exec_file) + assert self.config return ( [ "gcc", @@ -90,7 +91,6 @@ def execution(self, cwd: Path, file: str, arguments: List[str]) -> Command: return [str(local_file.absolute()), *arguments] def modify_solution(self, solution: Path): - # noinspection PyTypeChecker with open(solution, "r") as file: contents = file.read() # We use regex to find the main function. @@ -105,7 +105,6 @@ def modify_solution(self, solution: Path): with_args = re.compile(r"(int|void)(\s+)main(\s*)\((\s*)int") replacement = r"int\2solution_main\3(\4int" contents = re.sub(with_args, replacement, contents, count=1) - # noinspection PyTypeChecker with open(solution, "w") as file: header = "#pragma once\n\n" file.write(header + contents) @@ -114,6 +113,7 @@ def linter(self, remaining: float) -> Tuple[List[Message], List[AnnotateCode]]: # Import locally to prevent errors. from tested.languages.c import linter + assert self.config return linter.run_cppcheck(self.config.dodona, remaining) def cleanup_stacktrace(self, stacktrace: str) -> str: diff --git a/tested/languages/c/generators.py b/tested/languages/c/generators.py index 5fc57464..f12fc035 100644 --- a/tested/languages/c/generators.py +++ b/tested/languages/c/generators.py @@ -1,5 +1,5 @@ import json -from typing import List, Union +from typing import List, Union, cast from tested.datatypes import ( AdvancedNumericTypes, @@ -15,6 +15,7 @@ PreparedContext, PreparedExecutionUnit, PreparedTestcase, + PreparedTestcaseStatement, ) from tested.languages.utils import convert_unknown_type from tested.serialisation import ( @@ -26,20 +27,22 @@ NamedArgument, SpecialNumbers, Statement, + StringType, Value, VariableType, as_basic_type, ) -from tested.utils import get_args +from tested.testsuite import MainInput def convert_arguments(arguments: List[Expression | NamedArgument]) -> str: - return ", ".join(convert_statement(arg) for arg in arguments) + return ", ".join(convert_statement(cast(Expression, arg)) for arg in arguments) def convert_value(value: Value) -> str: # Handle some advanced types. if value.type == AdvancedStringTypes.CHAR: + assert isinstance(value, StringType) return f"(char) '" + value.data.replace("'", "\\'") + "'" elif value.type == AdvancedNumericTypes.INT_16: return f"((short) {value.data})" @@ -84,6 +87,7 @@ def convert_value(value: Value) -> str: elif value.type == BasicNothingTypes.NOTHING: return "NULL" elif value.type == BasicStringTypes.UNKNOWN: + assert isinstance(value, StringType) return convert_unknown_type(value) raise AssertionError(f"Invalid literal: {value!r}") @@ -143,9 +147,9 @@ def convert_statement(statement: Statement, full=False) -> str: return statement elif isinstance(statement, FunctionCall): return convert_function_call(statement) - elif isinstance(statement, get_args(Value)): + elif isinstance(statement, Value): return convert_value(statement) - elif isinstance(statement, get_args(Assignment)): + elif isinstance(statement, Assignment): if full: prefix = convert_declaration(statement.type) + " " else: @@ -170,6 +174,7 @@ def _generate_internal_context(ctx: PreparedContext, pu: PreparedExecutionUnit) result += f"{pu.execution_name}_write_separator();\n" if tc.testcase.is_main_testcase(): + assert isinstance(tc.input, MainInput) wrapped = [json.dumps(a) for a in tc.input.arguments] result += f'char* args[] = {{"{pu.submission_name}", ' result += ", ".join(wrapped) @@ -178,6 +183,7 @@ def _generate_internal_context(ctx: PreparedContext, pu: PreparedExecutionUnit) f"exit_code = solution_main({len(tc.input.arguments) + 1}, args);\n" ) else: + assert isinstance(tc.input, PreparedTestcaseStatement) result += "exit_code = 0;\n" result += convert_statement(tc.input.input_statement()) + ";\n" diff --git a/tested/languages/c/linter.py b/tested/languages/c/linter.py index 780a1e6f..c6512a09 100644 --- a/tested/languages/c/linter.py +++ b/tested/languages/c/linter.py @@ -79,7 +79,7 @@ def run_cppcheck( break annotations.append( AnnotateCode( - row=row, + row=row or 0, text=message, column=col, type=message_categories.get(severity, Severity.WARNING), diff --git a/tested/languages/config.py b/tested/languages/config.py index cc4ff11e..bf598d16 100644 --- a/tested/languages/config.py +++ b/tested/languages/config.py @@ -339,8 +339,20 @@ def filter_function(file: Path) -> bool: return list(x for x in files if filter_function(x)) + @typing.overload def find_main_file( - self, files: List[Path], name: str, precompilation_messages: List[str] + self, files: List[Path], name: str, precompilation_messages: List[Message] + ) -> Tuple[Path, List[Message], typing.Literal[Status.CORRECT], List[AnnotateCode]]: + ... + + @typing.overload + def find_main_file( + self, files: List[Path], name: str, precompilation_messages: List[Message] + ) -> Tuple[None, List[Message], Status, List[AnnotateCode]]: + ... + + def find_main_file( + self, files: List[Path], name: str, precompilation_messages: List[Message] ) -> Tuple[Optional[Path], List[Message], Status, List[AnnotateCode]]: """ Find the "main" file in a list of files. @@ -382,6 +394,7 @@ def cleanup_description(self, description: str) -> str: def get_description_generator(self) -> DescriptionGenerator: if self._description_generator is None: + assert self.config lang = self.config.dodona.programming_language config_dir = self.config.dodona.judge / "tested" / "languages" / lang self._description_generator = DescriptionGenerator(self, config_dir) @@ -445,5 +458,6 @@ def path_to_dependencies(self) -> List[Path]: :return: A list of template folders. """ + assert self.config lang = self.config.dodona.programming_language return [self.config.dodona.judge / "tested" / "languages" / lang / "templates"] diff --git a/tested/languages/conventionalize.py b/tested/languages/conventionalize.py index 7d965446..ffc7d960 100644 --- a/tested/languages/conventionalize.py +++ b/tested/languages/conventionalize.py @@ -416,6 +416,7 @@ def submission_name(language: "Language") -> str: """ :return: The name of a submission. """ + assert language.config return conventionalize_namespace(language, language.config.suite.namespace) diff --git a/tested/languages/csharp/config.py b/tested/languages/csharp/config.py index b59be8d5..e412e437 100644 --- a/tested/languages/csharp/config.py +++ b/tested/languages/csharp/config.py @@ -62,7 +62,7 @@ def supported_constructs(self) -> Set[Construct]: } def datatype_support(self) -> Mapping[AllTypes, TypeSupport]: - return { + return { # type: ignore "integer": "supported", "real": "supported", "char": "reduced", @@ -166,6 +166,7 @@ class {class_name} file.write(result) def cleanup_stacktrace(self, stacktrace: str) -> str: + assert self.config execution = conventionalize_namespace(self, EXECUTION_PREFIX) execution_submission_location_regex = ( rf"{self.config.dodona.workdir}/common/{execution}[0-9]+.cs" diff --git a/tested/languages/csharp/generators.py b/tested/languages/csharp/generators.py index 2a4b8fac..c41f880a 100644 --- a/tested/languages/csharp/generators.py +++ b/tested/languages/csharp/generators.py @@ -18,6 +18,7 @@ PreparedExecutionUnit, PreparedFunctionCall, PreparedTestcase, + PreparedTestcaseStatement, ) from tested.languages.utils import convert_unknown_type from tested.serialisation import ( @@ -27,13 +28,16 @@ FunctionType, Identifier, NamedArgument, + ObjectType, + SequenceType, SpecialNumbers, Statement, + StringType, Value, VariableType, as_basic_type, ) -from tested.utils import get_args +from tested.testsuite import MainInput def convert_arguments(arguments: List[Expression | NamedArgument]) -> str: @@ -49,8 +53,10 @@ def convert_arguments(arguments: List[Expression | NamedArgument]) -> str: def convert_value(value: Value) -> str: # Handle some advanced types. if value.type == AdvancedSequenceTypes.ARRAY: + assert isinstance(value, SequenceType) return f"new {convert_declaration(value.type, value)}{{{convert_arguments(value.data)}}}" elif value.type == AdvancedSequenceTypes.TUPLE: + assert isinstance(value, SequenceType) return f"({convert_arguments(value.data)})" elif value.type == AdvancedNumericTypes.SINGLE_PRECISION: if not isinstance(value.data, SpecialNumbers): @@ -101,8 +107,10 @@ def convert_value(value: Value) -> str: elif value.type == BasicNothingTypes.NOTHING: return "null" elif value.type in (BasicSequenceTypes.SEQUENCE, BasicSequenceTypes.SET): + assert isinstance(value, SequenceType) return f"new {convert_declaration(value.type, value)}() {{{convert_arguments(value.data)}}}" elif value.type == BasicObjectTypes.MAP: + assert isinstance(value, ObjectType) result = f"new {convert_declaration(value.type, value)}() {{" for i, pair in enumerate(value.data): result += ( @@ -113,6 +121,7 @@ def convert_value(value: Value) -> str: result += "}" return result elif value.type == BasicStringTypes.UNKNOWN: + assert isinstance(value, StringType) return convert_unknown_type(value) raise AssertionError(f"Invalid literal: {value!r}") @@ -152,7 +161,7 @@ def extract_type_tuple(type_, generic=True): # TODO: this is very complex, and why? type_ = ( (value.get_content_type() or "Object") - if isinstance(value, get_args(Value)) + if isinstance(value, SequenceType) else nt if nt else "Object" @@ -160,13 +169,15 @@ def extract_type_tuple(type_, generic=True): base_type, sub_type = extract_type_tuple(type_, False) return convert_declaration(base_type, None, sub_type) + "[]" elif tp == AdvancedSequenceTypes.TUPLE: + # TODO: rework this type_ = ( (value.get_content_type() or "Object") - if isinstance(value, get_args(Value)) + if isinstance(value, SequenceType) else nt if nt else "Object" ) + assert isinstance(value, SequenceType) base_type, sub_type = extract_type_tuple(type_, False) converted_type = convert_declaration(base_type, None, sub_type) return "(" + ", ".join(converted_type for _ in range(len(value.data))) @@ -194,7 +205,7 @@ def extract_type_tuple(type_, generic=True): if basic == BasicSequenceTypes.SEQUENCE: type_ = ( (value.get_content_type() or "Object") - if isinstance(value, get_args(Value)) + if isinstance(value, SequenceType) else nt if nt else "Object" @@ -204,7 +215,7 @@ def extract_type_tuple(type_, generic=True): elif basic == BasicSequenceTypes.SET: type_ = ( (value.get_content_type() or "Object") - if isinstance(value, get_args(Value)) + if isinstance(value, SequenceType) else nt if nt else "Object" @@ -220,7 +231,7 @@ def extract_type_tuple(type_, generic=True): elif basic == BasicNumericTypes.REAL: return "Double" elif basic == BasicObjectTypes.MAP: - if isinstance(value, get_args(Value)): + if isinstance(value, ObjectType): key_type_ = value.get_key_type() or "Object" value_type_ = value.get_value_type() or "Object" elif nt: @@ -240,9 +251,9 @@ def convert_statement(statement: Statement, full=False) -> str: return statement elif isinstance(statement, FunctionCall): return convert_function_call(statement) - elif isinstance(statement, get_args(Value)): + elif isinstance(statement, Value): return convert_value(statement) - elif isinstance(statement, get_args(Assignment)): + elif isinstance(statement, Assignment): if full: prefix = convert_declaration(statement.type, statement.expression) else: @@ -263,8 +274,10 @@ def _generate_internal_context(ctx: PreparedContext, pu: PreparedExecutionUnit) result += "WriteSeparator();\n" # Make a variable available outside of the try-catch block. - if not tc.testcase.is_main_testcase() and isinstance( - tc.input.statement, get_args(Assignment) + if ( + not tc.testcase.is_main_testcase() + and isinstance(tc.input, PreparedTestcaseStatement) + and isinstance(tc.input.statement, Assignment) ): result += ( convert_declaration( @@ -276,11 +289,13 @@ def _generate_internal_context(ctx: PreparedContext, pu: PreparedExecutionUnit) result += "try {" if tc.testcase.is_main_testcase(): + assert isinstance(tc.input, MainInput) result += " " * 4 + f"{pu.submission_name}.Main(new string[]{{" wrapped = [json.dumps(a) for a in tc.input.arguments] result += ", ".join(wrapped) result += "});\n" else: + assert isinstance(tc.input, PreparedTestcaseStatement) result += " " * 4 + convert_statement(tc.input.input_statement()) + ";\n" result += " " * 4 + convert_statement(tc.exception_statement()) + ";\n" result += "} catch (System.Exception E) {\n" diff --git a/tested/languages/description_generator.py b/tested/languages/description_generator.py index bc6c5f11..484570ef 100644 --- a/tested/languages/description_generator.py +++ b/tested/languages/description_generator.py @@ -75,16 +75,18 @@ def get_type_name( recursive_call: bool = False, ) -> str: programming_language = bundle.config.programming_language + if custom_type_map is None: + custom_type_map = dict() def _get_type(arg: str) -> Union[str, bool]: try: - return custom_type_map[programming_language][args] + return custom_type_map[programming_language][args] # type: ignore except KeyError: return self.types[arg] def _get_type_or_conventionalize(arg: str) -> str: try: - return _get_type(arg) + return _get_type(arg) # type: ignore except KeyError: return conventionalize_class(self.language, arg) @@ -93,16 +95,13 @@ def _get_type_name(arg: str) -> Union[str, bool]: return _get_type_or_conventionalize(arg) else: try: - return custom_type_map[programming_language]["inner"][arg] + return custom_type_map[programming_language]["inner"][arg] # type: ignore except KeyError: try: return self.types["inner"][arg] except KeyError: return _get_type_or_conventionalize(arg) - if custom_type_map is None: - custom_type_map = dict() - if isinstance(args, str): name = _get_type_name(args) type_name = name if isinstance(name, str) else args @@ -175,14 +174,18 @@ def get_global_variable_name(self, name: str, is_html: bool = True) -> str: return name def get_code( - self, stmt: str, bundle: Bundle, statement: bool = False, is_html: bool = True + self, + stmt_str: str, + bundle: Bundle, + statement: bool = False, + is_html: bool = True, ) -> str: from .generation import generate_statement if statement: - stmt = parse_string(stmt) + stmt = parse_string(stmt_str) else: - stmt = parse_string(stmt, is_return=True) + stmt = parse_string(stmt_str, is_return=True) required = stmt.get_used_features() available = self.language.supported_constructs() diff --git a/tested/languages/generation.py b/tested/languages/generation.py index c698a365..53ad22ac 100644 --- a/tested/languages/generation.py +++ b/tested/languages/generation.py @@ -38,8 +38,14 @@ Statement, Value, ) -from tested.testsuite import Context, FileUrl, ProgrammedEvaluator, Testcase, TextData -from tested.utils import get_args +from tested.testsuite import ( + Context, + FileUrl, + MainInput, + ProgrammedEvaluator, + Testcase, + TextData, +) # Prevent cyclic imports for types... if TYPE_CHECKING: @@ -101,6 +107,7 @@ def get_readable_input( if case.description: text = case.description elif case.is_main_testcase(): + assert isinstance(case.input, MainInput) # See https://rouge-ruby.github.io/docs/Rouge/Lexers/ConsoleLexer.html format_ = "console" arguments = " ".join(_escape_shell(x) for x in case.input.arguments) @@ -120,8 +127,8 @@ def get_readable_input( else: text = stdin else: + assert isinstance(case.input, Statement) format_ = bundle.config.programming_language - # noinspection PyTypeChecker text = generate_statement(bundle, case.input) text = bundle.lang_config.cleanup_description(text) analyse_files = True @@ -226,10 +233,10 @@ def generate_statement(bundle: Bundle, statement: Statement) -> str: :return: The code the statement. """ - if isinstance(statement, get_args(Expression)): + if isinstance(statement, Expression): statement = prepare_expression(bundle, statement) else: - assert isinstance(statement, get_args(Assignment)) + assert isinstance(statement, Assignment) statement = prepare_assignment(bundle, statement) return bundle.lang_config.generate_statement(statement) diff --git a/tested/languages/haskell/config.py b/tested/languages/haskell/config.py index 09e18105..50451ebc 100644 --- a/tested/languages/haskell/config.py +++ b/tested/languages/haskell/config.py @@ -41,7 +41,7 @@ def naming_conventions(self) -> Dict[Conventionable, NamingConventions]: } def datatype_support(self) -> Mapping[AllTypes, TypeSupport]: - return { + return { # type: ignore "integer": "supported", "real": "supported", "char": "supported", @@ -78,6 +78,7 @@ def supported_constructs(self) -> Set[Construct]: def compilation(self, files: List[str]) -> CallbackResult: main_ = files[-1] exec_ = main_.rstrip(".hs") + assert self.config return [ "ghc", "-fno-cse", @@ -99,12 +100,13 @@ def linter(self, remaining: float) -> Tuple[List[Message], List[AnnotateCode]]: # Import locally to prevent errors. from tested.languages.haskell import linter + assert self.config return linter.run_hlint(self.config.dodona, remaining) def cleanup_description(self, description: str) -> str: return cleanup_description(self, description) - def cleanup_stacktrace(self, traceback: str) -> str: + def cleanup_stacktrace(self, traceback_str: str) -> str: filename = submission_file(self) context_file_regex = re.compile(r"(Context[0-9]+|Selector)") compile_line_regex = re.compile(r"^([0-9]+)(\s*\|.*)$") @@ -113,7 +115,7 @@ def cleanup_stacktrace(self, traceback: str) -> str: ) parse_module = r"error: parse error on input ‘module’" replace_module = r"error: unexpected ‘module’" - traceback = traceback.splitlines() + traceback = traceback_str.splitlines() skip_line, lines = False, [] for line in traceback: if not line or line == "undefined": diff --git a/tested/languages/haskell/generators.py b/tested/languages/haskell/generators.py index 06c68117..820cd99f 100644 --- a/tested/languages/haskell/generators.py +++ b/tested/languages/haskell/generators.py @@ -17,6 +17,7 @@ PreparedContext, PreparedExecutionUnit, PreparedTestcase, + PreparedTestcaseStatement, ) from tested.languages.utils import convert_unknown_type from tested.serialisation import ( @@ -24,13 +25,16 @@ Expression, FunctionCall, Identifier, + NamedArgument, + SequenceType, SpecialNumbers, Statement, + StringType, Value, VariableType, as_basic_type, ) -from tested.utils import get_args +from tested.testsuite import MainInput def convert_arguments(arguments: List[Expression]) -> str: @@ -40,8 +44,9 @@ def convert_arguments(arguments: List[Expression]) -> str: def convert_value(value: Value) -> str: # Handle some advanced types. if value.type == AdvancedSequenceTypes.TUPLE: + assert isinstance(value, SequenceType) return f"({convert_arguments(value.data)})" - elif isinstance(value.type, get_args(AdvancedNumericTypes)): + elif isinstance(value.type, AdvancedNumericTypes): if not isinstance(value.data, SpecialNumbers): return f"{value.data} :: {convert_declaration(value.type)}" elif value.data == SpecialNumbers.NOT_A_NUMBER: @@ -52,6 +57,7 @@ def convert_value(value: Value) -> str: assert value.data == SpecialNumbers.NEG_INFINITY return f"(-1/0) :: {convert_declaration(value.type)}" elif value.type == AdvancedStringTypes.CHAR: + assert isinstance(value, StringType) return "'" + value.data.replace("'", "\\'") + "'" # Handle basic types value = as_basic_type(value) @@ -74,8 +80,10 @@ def convert_value(value: Value) -> str: elif value.type == BasicNothingTypes.NOTHING: return "Nothing :: Maybe Integer" elif value.type == BasicSequenceTypes.SEQUENCE: + assert isinstance(value, SequenceType) return f"[{convert_arguments(value.data)}]" elif value.type == BasicStringTypes.UNKNOWN: + assert isinstance(value, StringType) return convert_unknown_type(value) raise AssertionError(f"Invalid literal: {value!r}") @@ -86,9 +94,10 @@ def convert_function_call(function: FunctionCall) -> str: result += convert_statement(function.namespace) + "." result += function.name + " " for i, argument in enumerate(function.arguments): - if isinstance(argument, get_args(Value)): + if isinstance(argument, Value): result += convert_statement(argument) else: + assert not isinstance(argument, NamedArgument) result += "(" + convert_statement(argument) + ")" if i != len(function.arguments) - 1: result += " " @@ -137,7 +146,7 @@ def convert_declaration(tp: Union[AllTypes, VariableType]) -> str: def convert_statement(statement: Statement, lifting=False) -> str: - if isinstance(statement, get_args(Expression)): + if isinstance(statement, Expression): result = "" if lifting: result += "return (" @@ -146,7 +155,7 @@ def convert_statement(statement: Statement, lifting=False) -> str: elif isinstance(statement, FunctionCall): result += convert_function_call(statement) else: - assert isinstance(statement, get_args(Value)) + assert isinstance(statement, Value) result += convert_value(statement) if lifting: result += ")" @@ -230,6 +239,7 @@ def convert_execution_unit(pu: PreparedExecutionUnit) -> str: result += indent + "writeSeparator\n" if tc.testcase.is_main_testcase(): + assert isinstance(tc.input, MainInput) wrapped = [json.dumps(a) for a in tc.input.arguments] result += indent + f"let mainArgs = [{' '.join(wrapped)}]\n" result += ( @@ -241,9 +251,10 @@ def convert_execution_unit(pu: PreparedExecutionUnit) -> str: + f"let ee = handleException result in {convert_statement(tc.exception_statement('ee'))}\n" ) else: + assert isinstance(tc.input, PreparedTestcaseStatement) # In Haskell we do not actually have statements, so we need to keep them separate. # Additionally, exceptions with "statements" are not supported at this time. - if isinstance(tc.input.statement, get_args(Assignment)): + if isinstance(tc.input.statement, Assignment): result += indent + convert_statement(tc.input.statement) + "\n" else: result += indent + f"result{i1} <- catch\n" diff --git a/tested/languages/haskell/linter.py b/tested/languages/haskell/linter.py index 7da85020..cc89bc6d 100644 --- a/tested/languages/haskell/linter.py +++ b/tested/languages/haskell/linter.py @@ -26,17 +26,25 @@ def run_hlint( """ submission = config.source language_options = config.config_for() - if language_options.get("hlint_config", None): - config_path = config.resources / language_options.get("hlint_config") + if path := language_options.get("hlint_config", None): + assert isinstance(path, str) + config_path = config.resources / path else: # Use the default file. config_path = config.judge / "tested/languages/haskell/hlint.yml" - config_path = config_path.absolute() + config_path = str(config_path.absolute()) execution_results = run_command( directory=submission.parent, timeout=remaining, - command=["hlint", "-j", "--json", "-h", config_path, submission.absolute()], + command=[ + "hlint", + "-j", + "--json", + "-h", + config_path, + str(submission.absolute()), + ], ) if execution_results is None: diff --git a/tested/languages/java/config.py b/tested/languages/java/config.py index 8ecbf62c..cd8dc38b 100644 --- a/tested/languages/java/config.py +++ b/tested/languages/java/config.py @@ -55,7 +55,7 @@ def supported_constructs(self) -> Set[Construct]: } def datatype_support(self) -> Mapping[AllTypes, TypeSupport]: - return { + return { # type: ignore "integer": "supported", "real": "supported", "char": "supported", @@ -85,7 +85,7 @@ def datatype_support(self) -> Mapping[AllTypes, TypeSupport]: } def map_type_restrictions(self) -> Optional[Set[ExpressionTypes]]: - return { + return { # type: ignore "integer", "real", "char", @@ -118,6 +118,7 @@ def file_filter(file: Path) -> bool: return ["javac", "-cp", ".", *others], file_filter def execution(self, cwd: Path, file: str, arguments: List[str]) -> Command: + assert self.config limit = jvm_memory_limit(self.config) return ["java", f"-Xmx{limit}", "-cp", ".", Path(file).stem, *arguments] @@ -125,6 +126,7 @@ def linter(self, remaining: float) -> Tuple[List[Message], List[AnnotateCode]]: # Import locally to prevent errors. from tested.languages.java import linter + assert self.config return linter.run_checkstyle(self.config.dodona, remaining) def cleanup_stacktrace(self, traceback: str) -> str: diff --git a/tested/languages/java/generators.py b/tested/languages/java/generators.py index 2d621612..362b05eb 100644 --- a/tested/languages/java/generators.py +++ b/tested/languages/java/generators.py @@ -1,5 +1,5 @@ import json -from typing import List, Literal, Optional, Union +from typing import List, Literal, Optional, Union, cast from tested.datatypes import ( AdvancedNumericTypes, @@ -19,6 +19,7 @@ PreparedExecutionUnit, PreparedFunctionCall, PreparedTestcase, + PreparedTestcaseStatement, ) from tested.languages.utils import convert_unknown_type from tested.serialisation import ( @@ -27,13 +28,16 @@ FunctionCall, FunctionType, Identifier, + ObjectType, + SequenceType, SpecialNumbers, Statement, + StringType, Value, VariableType, as_basic_type, ) -from tested.utils import get_args +from tested.testsuite import MainInput def convert_arguments(arguments: List[Expression]) -> str: @@ -43,6 +47,7 @@ def convert_arguments(arguments: List[Expression]) -> str: def convert_value(value: Value) -> str: # Handle some advanced types. if value.type == AdvancedSequenceTypes.ARRAY: + assert isinstance(value, SequenceType) return f"new {convert_declaration(value.type, value)}{{{convert_arguments(value.data)}}}" elif value.type == AdvancedNumericTypes.SINGLE_PRECISION: if not isinstance(value.data, SpecialNumbers): @@ -73,6 +78,7 @@ def convert_value(value: Value) -> str: else: raise AssertionError("Special numbers not supported for BigDecimal") elif value.type == AdvancedStringTypes.CHAR: + assert isinstance(value, StringType) return "'" + value.data.replace("'", "\\'") + "'" # Handle basic types value = as_basic_type(value) @@ -99,10 +105,13 @@ def convert_value(value: Value) -> str: elif value.type == BasicNothingTypes.NOTHING: return "null" elif value.type == BasicSequenceTypes.SEQUENCE: + assert isinstance(value, SequenceType) return f"List.of({convert_arguments(value.data)})" elif value.type == BasicSequenceTypes.SET: + assert isinstance(value, SequenceType) return f"Set.of({convert_arguments(value.data)})" elif value.type == BasicObjectTypes.MAP: + assert isinstance(value, ObjectType) result = "Map.ofEntries(" for i, pair in enumerate(value.data): result += "Map.entry(" @@ -115,6 +124,7 @@ def convert_value(value: Value) -> str: result += ")" return result elif value.type == BasicStringTypes.UNKNOWN: + assert isinstance(value, StringType) return convert_unknown_type(value) raise AssertionError(f"Invalid literal: {value!r}") @@ -130,7 +140,7 @@ def convert_function_call(function: FunctionCall) -> str: result += convert_statement(function.namespace) + "." result += function.name if function.type != FunctionType.PROPERTY: - result += f"({convert_arguments(function.arguments)})" + result += f"({convert_arguments(cast(List[Expression], function.arguments))})" return result @@ -155,7 +165,7 @@ def extract_type_tuple(type_, generic=True): # TODO: this is very complex, and why? type_ = ( (value.get_content_type() or "Object") - if isinstance(value, get_args(Value)) + if isinstance(value, SequenceType) else nt if nt else "Object" @@ -187,7 +197,7 @@ def extract_type_tuple(type_, generic=True): if basic == BasicSequenceTypes.SEQUENCE: type_ = ( (value.get_content_type() or "Object") - if isinstance(value, get_args(Value)) + if isinstance(value, SequenceType) else nt if nt else "Object" @@ -197,7 +207,7 @@ def extract_type_tuple(type_, generic=True): elif basic == BasicSequenceTypes.SET: type_ = ( (value.get_content_type() or "Object") - if isinstance(value, get_args(Value)) + if isinstance(value, SequenceType) else nt if nt else "Object" @@ -213,7 +223,7 @@ def extract_type_tuple(type_, generic=True): elif basic == BasicNumericTypes.REAL: return "Double" if inner else "double" elif basic == BasicObjectTypes.MAP: - if isinstance(value, get_args(Value)): + if isinstance(value, ObjectType): key_type_ = value.get_key_type() or "Object" value_type_ = value.get_value_type() or "Object" elif nt: @@ -233,9 +243,9 @@ def convert_statement(statement: Statement, full=False) -> str: return statement elif isinstance(statement, FunctionCall): return convert_function_call(statement) - elif isinstance(statement, get_args(Value)): + elif isinstance(statement, Value): return convert_value(statement) - elif isinstance(statement, get_args(Assignment)): + elif isinstance(statement, Assignment): if full: prefix = convert_declaration(statement.type, statement.expression) + " " else: @@ -257,8 +267,10 @@ def _generate_internal_context(ctx: PreparedContext, pu: PreparedExecutionUnit) # In Java, we need special code to make variables available outside of # the try-catch block. - if not tc.testcase.is_main_testcase() and isinstance( - tc.input.statement, get_args(Assignment) + if ( + not tc.testcase.is_main_testcase() + and isinstance(tc.input, PreparedTestcaseStatement) + and isinstance(tc.input.statement, Assignment) ): result += ( convert_declaration( @@ -270,11 +282,13 @@ def _generate_internal_context(ctx: PreparedContext, pu: PreparedExecutionUnit) result += "try {\n" if tc.testcase.is_main_testcase(): + assert isinstance(tc.input, MainInput) result += " " * 4 + f"{pu.submission_name}.main(new String[]{{" wrapped = [json.dumps(a) for a in tc.input.arguments] result += ", ".join(wrapped) result += "});\n" else: + assert isinstance(tc.input, PreparedTestcaseStatement) result += " " * 4 + convert_statement(tc.input.input_statement()) + ";\n" result += " " * 4 + convert_statement(tc.exception_statement()) + ";\n" result += "} catch (Exception | AssertionError e) {\n" diff --git a/tested/languages/java/linter.py b/tested/languages/java/linter.py index 7af74648..5d84cab0 100644 --- a/tested/languages/java/linter.py +++ b/tested/languages/java/linter.py @@ -28,12 +28,13 @@ def run_checkstyle( submission = config.source language_options = config.config_for() - if language_options.get("checkstyle_config", None): - config_path = config.resources / language_options.get("checkstyle_config") + if path := language_options.get("checkstyle_config", None): + assert isinstance(path, str) + config_path = config.resources / path else: # Use the default file. config_path = config.judge / "tested/languages/java/sun_tested_checks.xml" - config_path = config_path.absolute() + config_path = str(config_path.absolute()) execution_results = run_command( directory=submission.parent, diff --git a/tested/languages/javascript/config.py b/tested/languages/javascript/config.py index 591e053e..ad2dd025 100644 --- a/tested/languages/javascript/config.py +++ b/tested/languages/javascript/config.py @@ -57,7 +57,7 @@ def supported_constructs(self) -> Set[Construct]: } def datatype_support(self) -> Mapping[AllTypes, TypeSupport]: - return { + return { # type: ignore "integer": "supported", "real": "supported", "char": "reduced", @@ -90,8 +90,7 @@ def map_type_restrictions(self) -> Optional[Set[ExpressionTypes]]: return {BasicStringTypes.TEXT} def set_type_restrictions(self) -> Optional[Set[ExpressionTypes]]: - # noinspection PyTypeChecker - return { + return { # type: ignore "integer", "real", "text", @@ -118,17 +117,19 @@ def modify_solution(self, solution: Path): # import local to prevent errors from tested.judge.utils import run_command - parse_file = Path(__file__).parent / "parseAst.js" + assert self.config + + parse_file = str(Path(__file__).parent / "parseAst.js") try: output = run_command( solution.parent, timeout=None, - command=["node", parse_file, solution.absolute()], + command=["node", parse_file, str(solution.absolute())], ) - namings = output.stdout.strip() - # noinspection PyTypeChecker - with open(solution, "a") as file: - print(f"\nmodule.exports = {{{namings}}};", file=file) + if output: + namings = output.stdout.strip() + with open(solution, "a") as file: + print(f"\nmodule.exports = {{{namings}}};", file=file) except TimeoutError: pass @@ -143,9 +144,11 @@ def linter(self, remaining: float) -> Tuple[List[Message], List[AnnotateCode]]: # Import locally to prevent errors. from tested.languages.javascript import linter + assert self.config return linter.run_eslint(self.config.dodona, remaining) def cleanup_stacktrace(self, traceback: str) -> str: + assert self.config # What this does: # 1a. While inside the submission code, replace all references to the location with # 1b. Remove any "submission.SOMETHING" -> "SOMETHING" diff --git a/tested/languages/javascript/generators.py b/tested/languages/javascript/generators.py index 47a38585..73c17086 100644 --- a/tested/languages/javascript/generators.py +++ b/tested/languages/javascript/generators.py @@ -1,5 +1,5 @@ import json -from typing import List +from typing import List, cast from tested.datatypes import ( AdvancedNothingTypes, @@ -15,6 +15,7 @@ PreparedContext, PreparedExecutionUnit, PreparedTestcase, + PreparedTestcaseStatement, ) from tested.languages.utils import convert_unknown_type from tested.serialisation import ( @@ -23,12 +24,15 @@ FunctionCall, FunctionType, Identifier, + ObjectType, + SequenceType, SpecialNumbers, Statement, + StringType, Value, as_basic_type, ) -from tested.utils import get_args +from tested.testsuite import MainInput def convert_arguments(arguments: List[Expression]) -> str: @@ -67,10 +71,13 @@ def convert_value(value: Value) -> str: elif value.type == BasicNothingTypes.NOTHING: return "null" elif value.type == BasicSequenceTypes.SEQUENCE: + assert isinstance(value, SequenceType) return f"[{convert_arguments(value.data)}]" elif value.type == BasicSequenceTypes.SET: + assert isinstance(value, SequenceType) return f"new Set([{convert_arguments(value.data)}])" elif value.type == BasicObjectTypes.MAP: + assert isinstance(value, ObjectType) result = "{" for i, pair in enumerate(value.data): result += convert_statement(pair.key, True) @@ -81,6 +88,7 @@ def convert_value(value: Value) -> str: result += "}" return result elif value.type == BasicStringTypes.UNKNOWN: + assert isinstance(value, StringType) return convert_unknown_type(value) raise AssertionError(f"Invalid literal: {value!r}") @@ -95,7 +103,7 @@ def convert_function_call(call: FunctionCall, internal=False) -> str: result += convert_statement(call.namespace, True) + "." result += call.name if call.type != FunctionType.PROPERTY: - result += f"({convert_arguments(call.arguments)})" + result += f"({convert_arguments(cast(List[Expression], call.arguments))})" return result @@ -104,9 +112,9 @@ def convert_statement(statement: Statement, internal=False, full=False) -> str: return statement elif isinstance(statement, FunctionCall): return convert_function_call(statement, internal) - elif isinstance(statement, get_args(Value)): + elif isinstance(statement, Value): return convert_value(statement) - elif isinstance(statement, get_args(Assignment)): + elif isinstance(statement, Assignment): if full: prefix = "let " else: @@ -135,6 +143,7 @@ def _generate_internal_context(ctx: PreparedContext, pu: PreparedExecutionUnit) # Prepare command arguments if needed. if tc.testcase.is_main_testcase(): + assert isinstance(tc.input, MainInput) wrapped = [json.dumps(a) for a in tc.input.arguments] result += f""" let new_args = [process.argv[0]]; @@ -143,18 +152,22 @@ def _generate_internal_context(ctx: PreparedContext, pu: PreparedExecutionUnit) """ # We need special code to make variables available outside of the try-catch block. - if not tc.testcase.is_main_testcase() and isinstance( - tc.input.statement, get_args(Assignment) + if ( + not tc.testcase.is_main_testcase() + and isinstance(tc.input, PreparedTestcaseStatement) + and isinstance(tc.input.statement, Assignment) ): result += f"let {tc.input.statement.variable}\n" result += "try {\n" if tc.testcase.is_main_testcase(): + assert isinstance(tc.input, MainInput) result += f""" delete require.cache[require.resolve("./{pu.submission_name}.js")]; const {pu.submission_name} = require("./{pu.submission_name}.js"); """ else: + assert isinstance(tc.input, PreparedTestcaseStatement) result += " " * 4 + convert_statement(tc.input.input_statement()) + ";\n" result += f""" diff --git a/tested/languages/javascript/linter.py b/tested/languages/javascript/linter.py index 4e42848a..382454b7 100644 --- a/tested/languages/javascript/linter.py +++ b/tested/languages/javascript/linter.py @@ -21,12 +21,13 @@ def run_eslint( """ submission = config.source language_options = config.config_for() - if language_options.get("eslint_config", None): - config_path = config.resources / language_options.get("eslint_config") + if path := language_options.get("eslint_config", None): + assert isinstance(path, str) + config_path = config.resources / path else: # Use the default file. config_path = config.judge / "tested/languages/javascript/eslintrc.yml" - config_path = config_path.absolute() + config_path = str(config_path.absolute()) execution_results = run_command( directory=submission.parent, @@ -38,7 +39,7 @@ def run_eslint( "--no-inline-config", "-c", config_path, - submission.absolute(), + str(submission.absolute()), ], ) diff --git a/tested/languages/kotlin/config.py b/tested/languages/kotlin/config.py index b2025d7b..ec1a4fb0 100644 --- a/tested/languages/kotlin/config.py +++ b/tested/languages/kotlin/config.py @@ -65,7 +65,7 @@ def supported_constructs(self) -> Set[Construct]: } def datatype_support(self) -> Mapping[AllTypes, TypeSupport]: - return { + return { # type: ignore "integer": "supported", "real": "supported", "char": "supported", @@ -96,7 +96,7 @@ def datatype_support(self) -> Mapping[AllTypes, TypeSupport]: } def map_type_restrictions(self) -> Optional[Set[ExpressionTypes]]: - return { + return { # type: ignore "integer", "real", "char", @@ -138,6 +138,7 @@ def file_filter(file: Path) -> bool: ], file_filter def execution(self, cwd: Path, file: str, arguments: List[str]) -> Command: + assert self.config limit = jvm_memory_limit(self.config) return [ get_executable("kotlin"), @@ -148,7 +149,6 @@ def execution(self, cwd: Path, file: str, arguments: List[str]) -> Command: *arguments, ] - # noinspection PyTypeChecker def modify_solution(self, solution: Path): with open(solution, "r") as file: contents = file.read() @@ -174,11 +174,12 @@ def linter(self, remaining: float) -> Tuple[List[Message], List[AnnotateCode]]: # Import locally to prevent errors. from tested.languages.kotlin import linter + assert self.config return linter.run_ktlint(self.config.dodona, remaining) def find_main_file( - self, files: List[Path], name: str, precompilation_messages: List[str] - ) -> Tuple[Optional[str], List[Message], Status, List[AnnotateCode]]: + self, files: List[Path], name: str, precompilation_messages: List[Message] + ) -> Tuple[Optional[Path], List[Message], Status, List[AnnotateCode]]: logger.debug("Finding %s in %s", name, files) main, msgs, status, ants = Language.find_main_file( self, files, name + "Kt", precompilation_messages @@ -189,10 +190,10 @@ def find_main_file( return Language.find_main_file(self, files, name, precompilation_messages) def filter_dependencies(self, files: List[Path], context_name: str) -> List[Path]: - def filter_function(file: Path) -> bool: + def filter_function(file_path: Path) -> bool: # We don't want files for contexts that are not the one we use. prefix = conventionalize_namespace(self, EXECUTION_PREFIX) - file = str(file) + file = str(file_path) is_context = file.startswith(prefix) is_our_context = file.startswith(context_name + ".") or file.startswith( context_name + "$" diff --git a/tested/languages/kotlin/generators.py b/tested/languages/kotlin/generators.py index 6f23700e..0a64a5fc 100644 --- a/tested/languages/kotlin/generators.py +++ b/tested/languages/kotlin/generators.py @@ -19,6 +19,7 @@ PreparedExecutionUnit, PreparedFunctionCall, PreparedTestcase, + PreparedTestcaseStatement, ) from tested.languages.utils import convert_unknown_type from tested.serialisation import ( @@ -28,13 +29,16 @@ FunctionType, Identifier, NamedArgument, + ObjectType, + SequenceType, SpecialNumbers, Statement, + StringType, Value, VariableType, as_basic_type, ) -from tested.utils import get_args +from tested.testsuite import MainInput def convert_arguments(arguments: List[Expression | NamedArgument]) -> str: @@ -50,6 +54,7 @@ def convert_arguments(arguments: List[Expression | NamedArgument]) -> str: def convert_value(value: Value) -> str: # Handle some advanced types. if value.type == AdvancedSequenceTypes.ARRAY: + assert isinstance(value, SequenceType) return f"arrayOf({convert_arguments(value.data)})" elif value.type == AdvancedNumericTypes.SINGLE_PRECISION: if not isinstance(value.data, SpecialNumbers): @@ -86,6 +91,7 @@ def convert_value(value: Value) -> str: else: raise AssertionError("Special numbers not supported for BigDecimal") elif value.type == AdvancedStringTypes.CHAR: + assert isinstance(value, StringType) return "'" + value.data.replace("'", "\\'") + "'" # Handle basic types value = as_basic_type(value) @@ -108,10 +114,13 @@ def convert_value(value: Value) -> str: elif value.type == BasicNothingTypes.NOTHING: return "null" elif value.type == BasicSequenceTypes.SEQUENCE: + assert isinstance(value, SequenceType) return f"listOf({convert_arguments(value.data)})" elif value.type == BasicSequenceTypes.SET: + assert isinstance(value, SequenceType) return f"setOf({convert_arguments(value.data)})" elif value.type == BasicObjectTypes.MAP: + assert isinstance(value, ObjectType) result = "mapOf(" for i, pair in enumerate(value.data): result += "Pair(" @@ -124,6 +133,7 @@ def convert_value(value: Value) -> str: result += ")" return result elif value.type == BasicStringTypes.UNKNOWN: + assert isinstance(value, StringType) return convert_unknown_type(value) raise AssertionError(f"Invalid literal: {value!r}") @@ -160,7 +170,7 @@ def extract_type_tuple(type_, generic=True): # TODO: this is very complex, and why? type_ = ( (value.get_content_type() or "Object") - if isinstance(value, get_args(Value)) + if isinstance(value, SequenceType) else nt if nt else "Object" @@ -192,7 +202,7 @@ def extract_type_tuple(type_, generic=True): if basic == BasicSequenceTypes.SEQUENCE: type_ = ( (value.get_content_type() or "Object") - if isinstance(value, get_args(Value)) + if isinstance(value, SequenceType) else nt if nt else "Object" @@ -202,7 +212,7 @@ def extract_type_tuple(type_, generic=True): elif basic == BasicSequenceTypes.SET: type_ = ( (value.get_content_type() or "Object") - if isinstance(value, get_args(Value)) + if isinstance(value, SequenceType) else nt if nt else "Object" @@ -218,7 +228,7 @@ def extract_type_tuple(type_, generic=True): elif basic == BasicNumericTypes.REAL: return "Double?" elif basic == BasicObjectTypes.MAP: - if isinstance(value, get_args(Value)): + if isinstance(value, ObjectType): key_type_ = value.get_key_type() or "Object" value_type_ = value.get_value_type() or "Object" elif nt: @@ -238,9 +248,9 @@ def convert_statement(statement: Statement, full=False) -> str: return statement elif isinstance(statement, FunctionCall): return convert_function_call(statement) - elif isinstance(statement, get_args(Value)): + elif isinstance(statement, Value): return convert_value(statement) - elif isinstance(statement, get_args(Assignment)): + elif isinstance(statement, Assignment): prefix = "var " if full else "" return ( f"{prefix}{statement.variable} = " @@ -313,8 +323,10 @@ class {pu.execution_name}: AutoCloseable {{ for tc in ctx.testcases: result += indent * 2 + "this.writeSeparator()\n" - if not tc.testcase.is_main_testcase() and isinstance( - tc.input.statement, get_args(Assignment) + if ( + not tc.testcase.is_main_testcase() + and isinstance(tc.input, PreparedTestcaseStatement) + and isinstance(tc.input.statement, Assignment) ): decl = convert_declaration( tc.input.statement.type, tc.input.statement.expression @@ -325,9 +337,11 @@ class {pu.execution_name}: AutoCloseable {{ result += indent * 2 + "try {\n" if tc.testcase.is_main_testcase(): + assert isinstance(tc.input, MainInput) wrapped = [json.dumps(a) for a in tc.input.arguments] result += indent * 3 + f"solutionMain(arrayOf({', '.join(wrapped)}))\n" else: + assert isinstance(tc.input, PreparedTestcaseStatement) result += ( indent * 3 + convert_statement(tc.input.input_statement()) + "\n" ) diff --git a/tested/languages/kotlin/linter.py b/tested/languages/kotlin/linter.py index 56e27c09..6bc94d86 100644 --- a/tested/languages/kotlin/linter.py +++ b/tested/languages/kotlin/linter.py @@ -21,13 +21,11 @@ def run_ktlint( submission = config.source language_options = config.config_for() - command = ["ktlint", "--reporter=json"] + command = ["ktlint", "--reporter=json", "--log-level=error"] - if language_options.get("editorconfig", None): - command.append( - "--editorconfig=" - f"{config.resources / language_options.get('editorconfig')}" - ) + if path := language_options.get("editorconfig", None): + assert isinstance(path, str) + command.append(f"--editorconfig={config.resources / path}") if language_options.get("disabled_rules_ktlint", None): rules = language_options["disabled_rules_ktlint"] @@ -39,17 +37,16 @@ def run_ktlint( else: command.append("--disabled_rules=filename") - if language_options.get("ktlint_ruleset", None): - command.append( - f"--ruleset={config.resources / language_options.get('ktlint_ruleset')}" - ) + if path := language_options.get("ktlint_ruleset", None): + assert isinstance(path, str) + command.append(f"--ruleset={config.resources / path}") if language_options.get("ktlint_experimental", True): command.append("--experimental") submission = submission.absolute() - command.append(submission.relative_to(submission.parent)) + command.append(str(submission.relative_to(submission.parent))) execution_results = run_command( directory=submission.parent, timeout=remaining, command=command @@ -74,6 +71,11 @@ def run_ktlint( ExtendedMessage( description=str(e), format="code", permission=Permission.STAFF ), + ExtendedMessage( + description=execution_results.stdout, + format="code", + permission=Permission.STAFF, + ), ], [] annotations = [] diff --git a/tested/languages/preparation.py b/tested/languages/preparation.py index d21e19ef..de44905b 100644 --- a/tested/languages/preparation.py +++ b/tested/languages/preparation.py @@ -6,7 +6,7 @@ """ from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Callable, List, Optional, Set, Tuple, Union +from typing import TYPE_CHECKING, Callable, List, Optional, Set, Tuple, Union, cast from tested.configs import Bundle from tested.languages.conventionalize import ( @@ -35,6 +35,7 @@ from tested.testsuite import ( Context, EmptyChannel, + EvaluatorOutputChannel, ExceptionOutput, IgnoredChannel, MainInput, @@ -43,7 +44,6 @@ TextData, ValueOutput, ) -from tested.utils import get_args if TYPE_CHECKING: from tested.judge.execution import ExecutionUnit @@ -83,11 +83,10 @@ def input_statement(self, override: Optional[str] = None) -> Statement: :return: The input statement. """ if self.value_function: - assert isinstance(self.statement, get_args(Expression)) + assert isinstance(self.statement, Expression) if override: return self.value_function(Identifier(override)) else: - # noinspection PyTypeChecker return self.value_function(self.statement) else: return self.statement @@ -265,15 +264,16 @@ def _create_handling_function( :return: A tuple containing the call and the name of the evaluator if present. """ lang_config = bundle.lang_config - if hasattr(output, "evaluator") and isinstance(output.evaluator, SpecificEvaluator): - # noinspection PyUnresolvedReferences + if isinstance(output, EvaluatorOutputChannel) and isinstance( + output.evaluator, SpecificEvaluator + ): evaluator = output.evaluator.for_language(bundle.config.programming_language) evaluator_name = conventionalize_namespace(lang_config, evaluator.file.stem) else: evaluator_name = None def generator(expression: Expression) -> Statement: - if hasattr(output, "evaluator") and isinstance( + if isinstance(output, EvaluatorOutputChannel) and isinstance( output.evaluator, SpecificEvaluator ): arguments = [ @@ -340,14 +340,12 @@ def prepare_testcase( names = [] if testcase.is_main_testcase(): - prepared_input = testcase.input + prepared_input = cast(MainInput, testcase.input) else: - if isinstance(testcase.input, get_args(Expression)): - # noinspection PyTypeChecker + if isinstance(testcase.input, Expression): command = prepare_expression(bundle, testcase.input) else: - assert isinstance(testcase.input, get_args(Assignment)) - # noinspection PyTypeChecker + assert isinstance(testcase.input, Assignment) command = prepare_assignment(bundle, testcase.input) result_channel = testcase.output.result diff --git a/tested/languages/python/config.py b/tested/languages/python/config.py index 99eaf26d..530cf726 100644 --- a/tested/languages/python/config.py +++ b/tested/languages/python/config.py @@ -59,7 +59,7 @@ def supported_constructs(self) -> Set[Construct]: } def datatype_support(self) -> Mapping[AllTypes, TypeSupport]: - return { + return { # type: ignore "integer": "supported", "real": "supported", "char": "reduced", @@ -90,8 +90,7 @@ def datatype_support(self) -> Mapping[AllTypes, TypeSupport]: } def map_type_restrictions(self) -> Optional[Set[ExpressionTypes]]: - # noinspection PyTypeChecker - return { + return { # type: ignore "integer", "real", "text", @@ -146,14 +145,15 @@ def linter(self, remaining: float) -> Tuple[List[Message], List[AnnotateCode]]: # Import locally to prevent errors. from tested.languages.python import linter + assert self.config return linter.run_pylint(self.config.dodona, remaining) # Idea and original code: dodona/judge-pythia - def cleanup_stacktrace(self, stacktrace: str) -> str: + def cleanup_stacktrace(self, stacktrace_str: str) -> str: context_file_regex = re.compile(r"context_[0-9]+_[0-9]+\.py") file_line_regex = re.compile(rf"\({submission_file(self)}, line (\d+)\)") file_full_regex = re.compile(rf'File "./{submission_file(self)}", line (\d+)') - stacktrace = stacktrace.splitlines(True) + stacktrace = stacktrace_str.splitlines(True) skip_line, lines = False, [] for line in stacktrace: diff --git a/tested/languages/python/generators.py b/tested/languages/python/generators.py index 8c0eee30..baafb9d9 100644 --- a/tested/languages/python/generators.py +++ b/tested/languages/python/generators.py @@ -16,6 +16,7 @@ PreparedExecutionUnit, PreparedFunctionCall, PreparedTestcase, + PreparedTestcaseStatement, ) from tested.languages.utils import convert_unknown_type from tested.serialisation import ( @@ -25,12 +26,15 @@ FunctionType, Identifier, NamedArgument, + ObjectType, + SequenceType, SpecialNumbers, Statement, + StringType, Value, as_basic_type, ) -from tested.utils import get_args +from tested.testsuite import MainInput def convert_arguments( @@ -48,6 +52,7 @@ def convert_arguments( def convert_value(value: Value) -> str: # Handle some advanced types. if value.type == AdvancedSequenceTypes.TUPLE: + assert isinstance(value, SequenceType) return f"({convert_arguments(value.data)})" elif value.type in ( AdvancedNumericTypes.DOUBLE_EXTENDED, @@ -81,10 +86,13 @@ def convert_value(value: Value) -> str: elif value.type == BasicNothingTypes.NOTHING: return "None" elif value.type == BasicSequenceTypes.SEQUENCE: + assert isinstance(value, SequenceType) return f"[{convert_arguments(value.data)}]" elif value.type == BasicSequenceTypes.SET: + assert isinstance(value, SequenceType) return f"{{{convert_arguments(value.data)}}}" elif value.type == BasicObjectTypes.MAP: + assert isinstance(value, ObjectType) result = "{" for i, pair in enumerate(value.data): result += convert_statement(pair.key, True) @@ -95,13 +103,17 @@ def convert_value(value: Value) -> str: result += "}" return result elif value.type == BasicStringTypes.UNKNOWN: + assert isinstance(value, StringType) return convert_unknown_type(value) raise AssertionError(f"Invalid literal: {value!r}") def convert_function_call(function: FunctionCall, with_namespace=False) -> str: result = "" - if function.namespace and (not function.has_root_namespace or with_namespace): + if function.namespace and ( + not (isinstance(function, PreparedFunctionCall) and function.has_root_namespace) + or with_namespace + ): result += convert_statement(function.namespace, with_namespace) + "." result += function.name if function.type != FunctionType.PROPERTY: @@ -114,9 +126,9 @@ def convert_statement(statement: Statement, with_namespace=False) -> str: return statement elif isinstance(statement, FunctionCall): return convert_function_call(statement, with_namespace) - elif isinstance(statement, get_args(Value)): + elif isinstance(statement, Value): return convert_value(statement) - elif isinstance(statement, get_args(Assignment)): + elif isinstance(statement, Assignment): return ( f"{statement.variable} = " f"{convert_statement(statement.expression, with_namespace)}" @@ -194,6 +206,7 @@ def send_specific_exception(exception): # Prepare command arguments if needed. if tc.testcase.is_main_testcase(): + assert isinstance(tc.input, MainInput) result += indent + "new_args = [sys.argv[0]]\n" wrapped = [json.dumps(a) for a in tc.input.arguments] result += f"{indent}new_args.extend([{', '.join(wrapped)}])\n" @@ -201,10 +214,12 @@ def send_specific_exception(exception): result += indent + "try:\n" if tc.testcase.is_main_testcase(): + assert isinstance(tc.input, MainInput) result += f"{indent*2}import {pu.submission_name}\n" if i != 0: result += f'{indent*2}importlib.reload(sys.modules["{pu.submission_name}"])\n' else: + assert isinstance(tc.input, PreparedTestcaseStatement) result += ( indent * 2 + convert_statement(tc.input.input_statement(), True) diff --git a/tested/languages/python/linter.py b/tested/languages/python/linter.py index 58964661..9e031e06 100644 --- a/tested/languages/python/linter.py +++ b/tested/languages/python/linter.py @@ -33,8 +33,9 @@ def run_pylint( """ submission = config.source language_options = config.config_for() - if language_options.get("pylint_config", None): - config_path = config.resources / language_options.get("pylint_config") + if path := language_options.get("pylint_config", None): + assert isinstance(path, str) + config_path = config.resources / path else: # Use the default file. config_path = config.judge / "tested/languages/python/pylint_config.rc" diff --git a/tested/languages/runhaskell/config.py b/tested/languages/runhaskell/config.py index e68030ab..b4d4af4a 100644 --- a/tested/languages/runhaskell/config.py +++ b/tested/languages/runhaskell/config.py @@ -29,12 +29,14 @@ def path_to_dependencies(self) -> List[Path]: :return: A list of template folders. """ + assert self.config return [ self.config.dodona.judge / "tested" / "languages" / "haskell" / "templates" ] def get_description_generator(self) -> DescriptionGenerator: if self._description_generator is None: + assert self.config config_dir = self.config.dodona.judge / "tested" / "languages" / "haskell" self._description_generator = DescriptionGenerator(self, config_dir) return self._description_generator diff --git a/tested/languages/utils.py b/tested/languages/utils.py index 4f45e674..4794fee5 100644 --- a/tested/languages/utils.py +++ b/tested/languages/utils.py @@ -34,14 +34,14 @@ def jvm_memory_limit(config: GlobalConfig) -> int: # Idea and original code: dodona/judge-pythia -def jvm_cleanup_stacktrace(stacktrace: str, submission_filename: str) -> str: +def jvm_cleanup_stacktrace(stacktrace_str: str, submission_filename: str) -> str: context_file_regex = re.compile(r"(Context[0-9]+|Selector)") execution_regex = re.compile(rf"Execution(\d+)\.kt") unresolved_main_regex = r"error: unresolved reference: solutionMain" unresolved_reference_regex = re.compile( r"(error: unresolved reference: [a-zA-Z$_0-9]+)" ) - stacktrace = stacktrace.splitlines(True) + stacktrace = stacktrace_str.splitlines(True) skip_line, lines = False, [] for line in stacktrace: @@ -84,6 +84,7 @@ def jvm_cleanup_stacktrace(stacktrace: str, submission_filename: str) -> str: def haskell_solution(lang_config: "Language", solution: Path): """Support implicit modules if needed.""" + assert lang_config.config if lang_config.config.dodona.config_for().get("implicitModule", True): name = submission_name(lang_config) # noinspection PyTypeChecker @@ -157,6 +158,7 @@ def convert_stacktrace_to_clickable_feedback( if not stacktrace: return None cleaned_stacktrace = lang.cleanup_stacktrace(stacktrace) + assert lang.config updated_stacktrace = _replace_code_line_number( lang.config.dodona.source_offset, cleaned_stacktrace ) diff --git a/tested/serialisation.py b/tested/serialisation.py index ab04dd69..d4af748c 100644 --- a/tested/serialisation.py +++ b/tested/serialisation.py @@ -25,7 +25,7 @@ from decimal import Decimal from enum import StrEnum, auto, unique from functools import reduce -from typing import Any, Iterable, List, Literal, Optional, Tuple, Union +from typing import Any, Iterable, List, Literal, Optional, Tuple, Union, cast from pydantic import BaseModel, Field, root_validator from pydantic.dataclasses import dataclass @@ -52,7 +52,7 @@ StringTypes, resolve_to_basic, ) -from tested.dodona import ExtendedMessage, Status +from tested.dodona import Message, Status from tested.features import Construct, FeatureSet, WithFeatures, combine_features from tested.utils import flatten, get_args, sorted_no_duplicates @@ -70,7 +70,7 @@ def _get_type_for(expression: "Expression") -> WrappedAllTypes: return expression.type, expression.get_content_type() elif isinstance(expression, ObjectType): return expression.type, (expression.get_key_type(), expression.get_value_type()) - elif isinstance(expression, get_args(Value)): + elif isinstance(expression, Value): return expression.type else: return BasicStringTypes.ANY @@ -497,18 +497,18 @@ def get_functions(self) -> Iterable["FunctionCall"]: Statement = Union[Assignment, Expression] # Update the forward references, which fixes the schema generation. -ObjectType.__pydantic_model__.update_forward_refs() -SequenceType.__pydantic_model__.update_forward_refs() -NamedArgument.__pydantic_model__.update_forward_refs() -FunctionCall.__pydantic_model__.update_forward_refs() -ObjectKeyValuePair.__pydantic_model__.update_forward_refs() +ObjectType.__pydantic_model__.update_forward_refs() # type: ignore +SequenceType.__pydantic_model__.update_forward_refs() # type: ignore +NamedArgument.__pydantic_model__.update_forward_refs() # type: ignore +FunctionCall.__pydantic_model__.update_forward_refs() # type: ignore +ObjectKeyValuePair.__pydantic_model__.update_forward_refs() # type: ignore def as_basic_type(value: Value) -> Value: """Convert a value's type to a basic type.""" new_type = resolve_to_basic(value.type) cp = copy.copy(value) - cp.type = new_type + cp.type = new_type # type: ignore return cp @@ -576,30 +576,41 @@ def _convert_to_python(value: Optional[Value], for_printing=False) -> Any: return None # If we have a type for which the data is usable in Python, use it. - if isinstance(value.type, get_args(SimpleTypes)): + if isinstance(value.type, SimpleTypes): # If we have floats or ints, convert them to Python. - if value.type in (NumericTypes.SINGLE_PRECISION, NumericTypes.DOUBLE_PRECISION): + if value.type in ( + AdvancedNumericTypes.SINGLE_PRECISION, + AdvancedNumericTypes.DOUBLE_PRECISION, + ): return float(str(value.data)) - if value.type != NumericTypes.FIXED_PRECISION: + if value.type != AdvancedNumericTypes.FIXED_PRECISION: return int(str(value.data)) if for_printing: + assert isinstance(value.data, Decimal) return PrintingDecimal(value.data) return value.data - if isinstance(value.type, get_args(SequenceTypes)): - values = [_convert_to_python(x) for x in value.data] + if isinstance(value.type, SequenceTypes): + assert isinstance(value, SequenceType) + values = [_convert_to_python(cast(Value, x)) for x in value.data] basic_type = resolve_to_basic(value.type) - if basic_type == SequenceTypes.SEQUENCE: + if basic_type == BasicSequenceTypes.SEQUENCE: return values - if basic_type == SequenceTypes.SET: - return sorted_no_duplicates(_convert_to_python(x) for x in value.data) + if basic_type == BasicSequenceTypes.SET: + return sorted_no_duplicates( + _convert_to_python(cast(Value, x)) for x in value.data + ) raise AssertionError(f"Unknown basic sequence type {basic_type}.") - if isinstance(value.type, get_args(ObjectTypes)): + if isinstance(value.type, ObjectTypes): + assert isinstance(value, ObjectType) return sorted_no_duplicates( ( - (_convert_to_python(pair.key), _convert_to_python(pair.value)) + ( + _convert_to_python(cast(Value, pair.key)), + _convert_to_python(cast(Value, pair.value)), + ) for pair in value.data ), key=lambda x: x[0], @@ -613,7 +624,7 @@ def _convert_to_python(value: Optional[Value], for_printing=False) -> Any: return str(value.data) -def serialize_from_python(value: Any, type_: str = None) -> Value: +def serialize_from_python(value: Any, type_: Optional[AllTypes] = None) -> Value: """ Convert a (simple) Python value into a TESTed value. @@ -621,14 +632,19 @@ def serialize_from_python(value: Any, type_: str = None) -> Value: JSON encoder for values of the language module for Python. """ if value is None: + assert isinstance(type_, NothingTypes | None) return NothingType(type=type_ or BasicNothingTypes.NOTHING) elif isinstance(value, str): + assert isinstance(type_, StringTypes | None) return StringType(type=type_ or BasicStringTypes.TEXT, data=value) elif isinstance(value, bool): + assert isinstance(type_, BooleanTypes | None) return BooleanType(type=type_ or BasicBooleanTypes.BOOLEAN, data=value) elif isinstance(value, int): + assert isinstance(type_, NumericTypes | None) return NumberType(type=type_ or BasicNumericTypes.INTEGER, data=value) elif isinstance(value, float): + assert isinstance(type_, NumericTypes | None) return NumberType(type=type_ or BasicNumericTypes.REAL, data=value) else: raise TypeError(f"No clue how to convert {value} into TESTed value.") @@ -657,7 +673,7 @@ def __bool__(self): return bool(self.value) -def to_python_comparable(value: Optional[Value]): +def to_python_comparable(value: Optional[Value]) -> Any: """ Convert the value into a comparable Python value. Most values are just converted to their built-in Python variant. Some, however, are not. For example, floats @@ -667,26 +683,37 @@ def to_python_comparable(value: Optional[Value]): the return channel (in the test suite). The returned value is only guaranteed to support the following operations: eq, str, repr and bool. """ - if value.type == AdvancedNumericTypes.FIXED_PRECISION: - return Decimal(value.data) - basic_type = resolve_to_basic(value.type) if value is None: return None + basic_type = resolve_to_basic(value.type) + if value.type == AdvancedNumericTypes.FIXED_PRECISION: + assert isinstance(value.data, Decimal) + return Decimal(value.data) if basic_type == BasicSequenceTypes.SEQUENCE: - return [to_python_comparable(x) for x in value.data] + assert isinstance(value, SequenceType) + return [to_python_comparable(cast(Value, x)) for x in value.data] if basic_type == BasicSequenceTypes.SET: - return sorted_no_duplicates(to_python_comparable(x) for x in value.data) + assert isinstance(value, SequenceType) + return sorted_no_duplicates( + to_python_comparable(cast(Value, x)) for x in value.data + ) if basic_type == BasicObjectTypes.MAP: + assert isinstance(value, ObjectType) return sorted_no_duplicates( ( - (to_python_comparable(pair.key), to_python_comparable(pair.value)) + ( + to_python_comparable(cast(Value, pair.key)), + to_python_comparable(cast(Value, pair.value)), + ) for pair in value.data ), key=lambda x: x[0], ) if basic_type == BasicNumericTypes.REAL: + assert isinstance(value, NumberType) return ComparableFloat(float(value.data)) if basic_type == BasicNumericTypes.INTEGER: + assert isinstance(value, NumberType) return value.data if basic_type in ( BasicBooleanTypes.BOOLEAN, @@ -701,15 +728,42 @@ def to_python_comparable(value: Optional[Value]): class EvalResult(BaseModel): - result: Union[bool, Status] + result: Status + readable_expected: Optional[str] = Field(None, alias="readableExpected") + readable_actual: Optional[str] = Field(None, alias="readableActual") + messages: List[Message] = Field(default_factory=list) + + class Config: + # Allow both camel case and snake case fields + allow_population_by_field_name = True + + +class BooleanEvalResult(BaseModel): + """ + Allows a boolean result. + """ + + result: Status | bool readable_expected: Optional[str] = Field(None, alias="readableExpected") readable_actual: Optional[str] = Field(None, alias="readableActual") - messages: List[ExtendedMessage] = Field(default_factory=list) + messages: List[Message] = Field(default_factory=list) class Config: # Allow both camel case and snake case fields allow_population_by_field_name = True + def as_eval_result(self) -> EvalResult: + if isinstance(self.result, Status): + status = self.result + else: + status = Status.CORRECT if self.result else Status.WRONG + return EvalResult( + result=status, + readable_expected=self.readable_expected, # type: ignore + readable_actual=self.readable_actual, # type: ignore + messages=self.messages, + ) + @dataclass class ExceptionValue: diff --git a/tested/testsuite.py b/tested/testsuite.py index 9af219ed..b46fea54 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -27,12 +27,13 @@ Expression, FunctionCall, FunctionType, + NamedArgument, SequenceType, Statement, Value, WithFunctions, ) -from tested.utils import flatten, get_args +from tested.utils import flatten @unique @@ -343,23 +344,21 @@ def get_used_features(self) -> FeatureSet: return NOTHING -SpecialOutputChannel = Union[EmptyChannel, IgnoredChannel] +SpecialOutputChannel = EmptyChannel | IgnoredChannel -NormalOutputChannel = Union[ - TextOutputChannel, - FileOutputChannel, - ValueOutputChannel, - ExceptionOutputChannel, - ExitCodeOutputChannel, +EvaluatorOutputChannel = Union[ + TextOutputChannel, FileOutputChannel, ValueOutputChannel, ExceptionOutputChannel ] -OutputChannel = Union[NormalOutputChannel, SpecialOutputChannel] +NormalOutputChannel = EvaluatorOutputChannel | ExitCodeOutputChannel -TextOutput = Union[TextOutputChannel, SpecialOutputChannel] -FileOutput = Union[FileOutputChannel, IgnoredChannel] -ExceptionOutput = Union[ExceptionOutputChannel, SpecialOutputChannel] -ValueOutput = Union[ValueOutputChannel, SpecialOutputChannel] -ExitOutput = Union[ExitCodeOutputChannel, IgnoredChannel] +OutputChannel = NormalOutputChannel | SpecialOutputChannel + +TextOutput = TextOutputChannel | SpecialOutputChannel +FileOutput = FileOutputChannel | IgnoredChannel +ExceptionOutput = ExceptionOutputChannel | SpecialOutputChannel +ValueOutput = ValueOutputChannel | SpecialOutputChannel +ExitOutput = ExitCodeOutputChannel | IgnoredChannel @dataclass @@ -393,7 +392,9 @@ def get_specific_eval_languages(self) -> Optional[Set[str]]: if isinstance(self.exception, ExceptionOutputChannel): if isinstance(self.exception.evaluator, SpecificEvaluator): languages = set(self.exception.evaluator.evaluators.keys()) - elif self.exception.exception.types: + elif ( + self.exception.exception is not None and self.exception.exception.types + ): languages = set(self.exception.exception.types.keys()) if isinstance(self.result, ValueOutputChannel): if isinstance(self.result.evaluator, SpecificEvaluator): @@ -416,9 +417,9 @@ class MainInput(WithFeatures, WithFunctions): arguments: List[str] = field(default_factory=list) main_call: bool = True # Deprecated, to remove... - def get_as_string(self, working_directory): + def get_as_string(self, working_directory: Path) -> str: if self.stdin == EmptyChannel.NONE: - return None + return "" else: return self.stdin.get_data_as_string(working_directory) @@ -481,7 +482,7 @@ def no_return_with_assignment(cls, values): # this is an error: a statement is not an expression. output = values["output"] if output.result != EmptyChannel.NONE and not isinstance( - values["input"], get_args(Expression) + values["input"], Expression ): raise ValueError("You cannot expect a value from a statement.") return values @@ -524,7 +525,8 @@ def get_functions(self) -> Iterable[FunctionCall]: def get_stdin(self, resources: Path) -> str: first_testcase = self.testcases[0] - if isinstance(first_testcase.input, MainInput): + if self.has_main_testcase(): + assert isinstance(first_testcase.input, MainInput) return first_testcase.input.get_as_string(resources) else: return "" @@ -553,12 +555,15 @@ class Tab(WithFeatures, WithFunctions): runs: Optional[list] = None def get_used_features(self) -> FeatureSet: + assert self.contexts is not None return combine_features(x.get_used_features() for x in self.contexts) def get_functions(self) -> Iterable[FunctionCall]: + assert self.contexts is not None return flatten(x.get_functions() for x in self.contexts) def count_contexts(self): + assert self.contexts is not None return len(self.contexts) @root_validator(pre=True) @@ -594,8 +599,8 @@ def must_have_contexts(cls, values): def unique_evaluation_functions(cls, contexts: List[Context]) -> List[Context]: eval_functions: Dict[str, List[EvaluationFunction]] = defaultdict(list) - for context in contexts: # type: Context - for testcase in context.testcases: # type: Testcase + for context in contexts: + for testcase in context.testcases: output = testcase.output if isinstance(output.result, ValueOutputChannel) and isinstance( output.result.evaluator, SpecificEvaluator @@ -724,19 +729,20 @@ def _resolve_function_calls(function_calls: Iterable[FunctionCall]): argument_map: Dict[Any, List[Expression]] = defaultdict(list) for call in calls: for i, arg in enumerate(call.arguments): + if isinstance(arg, NamedArgument): + arg = arg.value argument_map[i].append(arg) # All types inside the every list should be the same. # TODO: this has some limitations, more specifically, function calls and # identifiers are not checked, as it is not known which types they are. type_use = [] - # noinspection PyTypeChecker for arguments in argument_map.values(): types = set() for argument in arguments: if isinstance(argument, SequenceType): types.add((argument.type, argument.get_content_type())) - elif isinstance(argument, get_args(Value)): + elif isinstance(argument, Value): types.add(argument.type) type_use.append(types) if not all(len(x) <= 1 for x in type_use): @@ -763,7 +769,7 @@ def generate_schema(): """ import json - sc = Suite.__pydantic_model__.schema() + sc = _SuiteModel.schema() sc["$schema"] = "http://json-schema.org/draft-07/schema#" print(json.dumps(sc, indent=2)) diff --git a/tested/utils.py b/tested/utils.py index bbfba654..0b79b8a7 100644 --- a/tested/utils.py +++ b/tested/utils.py @@ -1,28 +1,13 @@ import contextlib import itertools import logging -import os import random -import stat import string import sys import typing from itertools import zip_longest -from os import PathLike from pathlib import Path -from typing import ( - IO, - Any, - Callable, - Generator, - Generic, - Iterable, - List, - Mapping, - Optional, - TypeVar, - Union, -) +from typing import IO, Any, Callable, Iterable, List, Optional, TypeVar, Union _logger = logging.getLogger(__name__) @@ -39,19 +24,6 @@ def smart_close(file: IO): return contextlib.nullcontext(file) -@contextlib.contextmanager -def protected_directory( - directory: Union[PathLike, Path] -) -> Generator[Path, None, None]: - try: - _logger.info("Making %s read-only", directory) - os.chmod(directory, stat.S_IREAD) # Disable write access - yield directory - finally: - _logger.info("Giving write-access to %s", directory) - os.chmod(directory, stat.S_IREAD | stat.S_IWRITE) - - def basename(file: Union[str, Path]) -> str: """ Get the basename of a file. @@ -86,7 +58,6 @@ def consume_shebang(submission: Path) -> Optional[str]: """ language = None try: - # noinspection PyTypeChecker with open(submission, "r+") as file: lines = file.readlines() file.seek(0) @@ -113,24 +84,10 @@ def consume_shebang(submission: Path) -> Optional[str]: K = TypeVar("K") -V = TypeVar("V") T = TypeVar("T") -class _FallbackDict(dict, Generic[K, V]): - def __init__(self, existing: Mapping[K, V], fallback_: Mapping[K, V]): - super().__init__(existing) - self.fallback = fallback_ - - def __missing__(self, key: K) -> Optional[V]: - return self.fallback[key] - - -def fallback(source: Mapping[K, V], additions: Mapping[K, V]) -> Mapping[K, V]: - return _FallbackDict(additions, source) - - -def get_args(type_): +def get_args(type_: Any) -> tuple[Any, ...]: """ Get the args of a type or the type itself. @@ -238,13 +195,9 @@ def recursive_dict_merge(one: dict, two: dict) -> dict: def sorted_no_duplicates( iterable: Iterable[T], - key: Optional[Callable[[T], K]] = None, + key: Callable[[T], K] = lambda x: x, recursive_key: Optional[Callable[[K], K]] = None, ) -> List[T]: - # Identity key function - def identity(x: T) -> T: - return x - # Order functions def type_order(x: Any, y: Any) -> int: """ @@ -356,10 +309,6 @@ def timsort(list_t: List[T], timgroup: int = 32) -> List[T]: timgroup *= 2 return copy - # Check if key function is not none - if key is None: - key = identity - # Sort and filterout duplicates first, last_key, no_dup, list_iter = True, None, [], list(iterable) for v in timsort(list_iter): diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index 147cf1dc..e51e3299 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -35,7 +35,6 @@ def test_parse_one_tab_ctx(): namespace: "solution" tabs: - tab: "Ctx" - hidden: true testcases: - arguments: [ "--arg", "argument" ] stdin: "Input string" @@ -48,7 +47,6 @@ def test_parse_one_tab_ctx(): assert suite.namespace == "solution" assert len(suite.tabs) == 1 tab = suite.tabs[0] - assert tab.hidden assert tab.name == "Ctx" assert len(tab.contexts) == 1 context = tab.contexts[0] diff --git a/tests/test_evaluators.py b/tests/test_evaluators.py index d100632f..aac94489 100644 --- a/tests/test_evaluators.py +++ b/tests/test_evaluators.py @@ -7,13 +7,13 @@ import tested from tested.configs import create_bundle -from tested.datatypes import BasicStringTypes +from tested.datatypes import BasicSequenceTypes, BasicStringTypes from tested.dodona import Status from tested.evaluators.common import EvaluationResult, EvaluatorConfig from tested.evaluators.exception import evaluate as evaluate_exception from tested.evaluators.text import evaluate_file, evaluate_text from tested.evaluators.value import evaluate as evaluate_value -from tested.serialisation import ExceptionValue, StringType +from tested.serialisation import ExceptionValue, SequenceType, StringType from tested.testsuite import ( ExceptionOutputChannel, ExpectedException, @@ -431,12 +431,62 @@ def test_value_string_as_text_is_detected_when_no_actual(tmp_path: Path, pytestc channel = ValueOutputChannel( value=StringType(type=BasicStringTypes.TEXT, data="multi\nline\nstring") ) - actual_value = json.dumps( - StringType(type=BasicStringTypes.TEXT, data="multi\nline\nstring"), - default=pydantic_encoder, - ) config = evaluator_config(tmp_path, pytestconfig, language="python") result = evaluate_value(config, channel, "") assert result.result.enum == Status.WRONG assert result.readable_expected == "multi\nline\nstring" assert result.readable_actual == "" + + +def test_nested_sets_type_check_works_if_correct(tmp_path: Path, pytestconfig): + expected_value = SequenceType( + type=BasicSequenceTypes.SET, + data=[ + SequenceType( + type=BasicSequenceTypes.SET, + data=[ + StringType(type=BasicStringTypes.TEXT, data="a"), + StringType(type=BasicStringTypes.TEXT, data="b"), + ], + ), + SequenceType( + type=BasicSequenceTypes.SET, + data=[ + StringType(type=BasicStringTypes.TEXT, data="a"), + StringType(type=BasicStringTypes.TEXT, data="a"), + ], + ), + StringType(type=BasicStringTypes.TEXT, data="c"), + ], + ) + actual_value = SequenceType( + type=BasicSequenceTypes.SET, + data=[ + SequenceType( + type=BasicSequenceTypes.SET, + data=[ + StringType(type=BasicStringTypes.TEXT, data="a"), + StringType(type=BasicStringTypes.TEXT, data="a"), + ], + ), + StringType(type=BasicStringTypes.TEXT, data="c"), + SequenceType( + type=BasicSequenceTypes.SET, + data=[ + StringType(type=BasicStringTypes.TEXT, data="b"), + StringType(type=BasicStringTypes.TEXT, data="a"), + ], + ), + ], + ) + channel = ValueOutputChannel(value=expected_value) + config = evaluator_config(tmp_path, pytestconfig, language="python") + result = evaluate_value( + config, + channel, + json.dumps( + actual_value, + default=pydantic_encoder, + ), + ) + assert result.result.enum == Status.CORRECT diff --git a/tests/test_functionality.py b/tests/test_functionality.py index c457e2c9..c0276d9d 100644 --- a/tests/test_functionality.py +++ b/tests/test_functionality.py @@ -605,7 +605,7 @@ def test_batch_compilation_no_fallback( result = execute_config(conf) updates = assert_valid_output(result, pytestconfig) assert len(updates.find_all("start-tab")) == 1 - assert updates.find_status_enum() == ["compilation error"] * 2 + assert updates.find_status_enum() == ["compilation error"] assert spy.call_count == 1 diff --git a/tests/test_serialisation.py b/tests/test_serialisation.py index 79a991eb..c7e8a86c 100644 --- a/tests/test_serialisation.py +++ b/tests/test_serialisation.py @@ -33,7 +33,7 @@ BasicTypes, resolve_to_basic, ) -from tested.evaluators.value import check_data_type +from tested.evaluators.value import _check_simple_type from tested.features import TypeSupport, fallback_type_support_map from tested.judge.compilation import run_compilation from tested.judge.execution import execute_file, filter_files @@ -224,7 +224,7 @@ def test_basic_types(language, tmp_path: Path, pytestconfig): for result, expected in zip(results, types): actual = parse_value(result) - type_check, _ = check_data_type(bundle, expected, actual) + type_check, _ = _check_simple_type(bundle, expected, actual) assert type_check, f"type check failure {expected} != {actual}" py_expected = to_python_comparable(expected) py_actual = to_python_comparable(actual) @@ -250,7 +250,7 @@ def test_advanced_types(language, tmp_path: Path, pytestconfig): for result, expected in zip(results, types): actual = parse_value(result) - type_check, _ = check_data_type(bundle, expected, actual) + type_check, _ = _check_simple_type(bundle, expected, actual) assert type_check, f"type check failure {expected} != {actual}" py_expected = to_python_comparable(expected) py_actual = to_python_comparable(actual) @@ -305,7 +305,7 @@ def test_special_numbers(language, tmp_path: Path, pytestconfig): for result, expected in zip(results, types): actual = parse_value(result) - type_check, _ = check_data_type(bundle, expected, actual) + type_check, _ = _check_simple_type(bundle, expected, actual) assert type_check, f"type check failure {expected} != {actual}" py_expected = to_python_comparable(expected) py_actual = to_python_comparable(actual) From f5d42727b51b021121ba8809fcf363fd3c695f7c Mon Sep 17 00:00:00 2001 From: Niko Strijbol Date: Mon, 7 Aug 2023 16:09:33 +0200 Subject: [PATCH 2/2] Add type checker to GA --- .github/workflows/ci.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b77c364e..b5f74a49 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,3 +54,19 @@ jobs: with: version: "~= 23.0" src: "./tested ./tests" + types: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: 3.11.2 + cache: 'pipenv' + - run: pip install pipenv + - run: pipenv install --dev + - run: echo "$(pipenv --venv)/bin" >> $GITHUB_PATH + - uses: jakebailey/pyright-action@v1 + with: + version: '1.1.316' + warnings: true + working-directory: tested/