diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index 6cd69cc9..0dc4dc75 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -350,7 +350,6 @@ { "required" : [ "oracle", - "language", "file" ], "properties" : { @@ -360,10 +359,6 @@ "custom_check" ] }, - "language" : { - "type" : "string", - "description" : "The programming language of the custom check function." - }, "file" : { "type" : "string", "description" : "The path to the file containing the custom check function." @@ -415,7 +410,6 @@ { "required" : [ "oracle", - "language", "file" ], "properties" : { @@ -425,10 +419,6 @@ "custom_check" ] }, - "language" : { - "type" : "string", - "description" : "The programming language of the custom check function." - }, "file" : { "type" : "string", "description" : "The path to the file containing the custom check function." diff --git a/tested/dsl/schema_draft7.json b/tested/dsl/schema_draft7.json index 154ee4e9..dccd8036 100644 --- a/tested/dsl/schema_draft7.json +++ b/tested/dsl/schema_draft7.json @@ -345,7 +345,6 @@ { "required" : [ "oracle", - "language", "file" ], "properties" : { @@ -355,10 +354,6 @@ "custom_check" ] }, - "language" : { - "type" : "string", - "description" : "The programming language of the custom check function." - }, "file" : { "type" : "string", "description" : "The path to the file containing the custom check function." @@ -409,7 +404,6 @@ { "required" : [ "oracle", - "language", "file" ], "properties" : { @@ -419,10 +413,6 @@ "custom_check" ] }, - "language" : { - "type" : "string", - "description" : "The programming language of the custom check function." - }, "file" : { "type" : "string", "description" : "The path to the file containing the custom check function." diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 18960eb5..16324483 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -343,7 +343,6 @@ def _convert_custom_check_oracle(stream: dict) -> CustomCheckOracle: assert isinstance(cv, Value) converted_args.append(cv) return CustomCheckOracle( - language=stream["language"], function=EvaluationFunction( file=stream["file"], name=stream.get("name", "evaluate") ), diff --git a/tested/judge/programmed.py b/tested/judge/programmed.py deleted file mode 100644 index 379184e6..00000000 --- a/tested/judge/programmed.py +++ /dev/null @@ -1,273 +0,0 @@ -""" -Programmed evaluation. -""" -import contextlib -import logging -import shutil -import sys -import types -from collections.abc import Generator -from io import StringIO -from pathlib import Path -from typing import cast - -from tested.configs import Bundle, create_bundle -from tested.dodona import ExtendedMessage, Permission, Status -from tested.features import Construct -from tested.internationalization import get_i18n_string -from tested.judge.execution import execute_file, filter_files -from tested.judge.utils import BaseExecutionResult, copy_from_paths_to_path, run_command -from tested.languages.generation import generate_custom_evaluator, generate_statement -from tested.oracles.common import BooleanEvalResult -from tested.serialisation import FunctionCall, FunctionType, Value -from tested.testsuite import CustomCheckOracle -from tested.utils import get_identifier - -_logger = logging.getLogger(__name__) - - -def evaluate_programmed( - bundle: Bundle, - evaluator: CustomCheckOracle, - expected: Value, - actual: Value, -) -> BaseExecutionResult | BooleanEvalResult: - """ - Run the custom evaluation. Concerning structure and execution, the custom - oracle is very similar to the execution of the whole evaluation. It a - mini-evaluation if you will. - """ - - _logger.debug("Doing programmed output") - - # We have special support for Python. - if evaluator.language == "python" and bundle.config.options.optimized: - return _evaluate_python(bundle, evaluator, expected, actual) - else: - return _evaluate_others(bundle, evaluator, expected, actual) - - -def _evaluate_others( - bundle: Bundle, - evaluator: CustomCheckOracle, - expected: Value, - actual: Value, -) -> BaseExecutionResult: - """ - Evaluate in all languages but Python. The re-uses the infrastructure of the - templates and execution facilities of the language config. - """ - _logger.debug("Doing evaluation in non-Python mode.") - - # Create a directory for this oracle. If one exists, delete it first. - evaluator_dir_name = evaluator.function.file.stem - custom_directory_name = f"{get_identifier()}_{evaluator_dir_name}" - custom_path = Path(bundle.config.workdir, "functions", custom_directory_name) - if custom_path.exists(): - _logger.debug("Removing existing directory for custom oracle.") - shutil.rmtree(custom_path, ignore_errors=True) - custom_path.mkdir(parents=True) - - _logger.info("Will do custom evaluation in %s", custom_path) - - # Create a configs bundle for the language of the oracle. - eval_bundle = create_bundle( - bundle.config, bundle.out, bundle.suite, evaluator.language - ) - - # Check if the language supports this. - if Construct.EVALUATION not in eval_bundle.language.supported_constructs(): - _logger.error( - f"{eval_bundle.config.programming_language} does not support" - f" evaluations." - ) - return BaseExecutionResult( - stdout="", - stderr=get_i18n_string( - "judge.programmed.unsupported", - lang=eval_bundle.config.programming_language, - ), - exit=-1, - timeout=False, - memory=False, - ) - - # Copy the oracle - origin_path = Path(bundle.config.resources, evaluator.function.file) - _logger.debug("Copying %s to %s", origin_path, custom_path) - shutil.copy2(origin_path, custom_path) - - # Copy the dependencies to the folder. - dependencies = eval_bundle.language.initial_dependencies() - origin = eval_bundle.language.path_to_dependencies() - copy_from_paths_to_path(origin, dependencies, custom_path) - # Include the actual oracle in the dependencies. - dependencies.append(evaluator.function.file.name) - - # Generate the oracle. - _logger.debug("Generating custom oracle.") - evaluator_name = generate_custom_evaluator( - eval_bundle, - destination=custom_path, - evaluator=evaluator, - expected_value=expected, - actual_value=actual, - ) - dependencies.append(evaluator_name) - _logger.debug("Generated oracle executor %s", evaluator_name) - - # Do compilation for those configs that require it. - command, files = eval_bundle.language.compilation(dependencies) - _logger.debug("Compiling custom oracle with command %s", command) - result = run_command(custom_path, None, command) - if result and result.stderr: - raise ValueError("Error while compiling specific test case:" + result.stderr) - - files = filter_files(files, custom_path) - # Execute the custom oracle. - evaluator_name = Path(evaluator_name).stem - - files = eval_bundle.language.filter_dependencies(files, evaluator_name) - for file in files: - origin = custom_path / file - try: - shutil.copy2(origin, custom_path) - except shutil.SameFileError: - # If the file already exists, skip it. - pass - - executable, status = eval_bundle.language.find_main_file(files, evaluator_name) - - if status != Status.CORRECT: - return BaseExecutionResult( - stdout="", - stderr=get_i18n_string("judge.programmed.unknown.compilation"), - exit=-1, - timeout=False, - memory=False, - ) - - files.remove(executable) - - return execute_file( - bundle=eval_bundle, - executable_name=executable.name, - working_directory=custom_path, - stdin=None, - remaining=None, - ) - - -@contextlib.contextmanager -def _catch_output() -> Generator[tuple[StringIO, StringIO], None, None]: - old_stdout = sys.stdout - old_stderr = sys.stderr - stdout = StringIO() - stderr = StringIO() - try: - sys.stdout = stdout - sys.stderr = stderr - yield stdout, stderr - finally: - sys.stdout = old_stdout - sys.stderr = old_stderr - - -def _evaluate_python( - bundle: Bundle, - oracle: CustomCheckOracle, - expected: Value, - actual: Value, -) -> BooleanEvalResult: - """ - Run an evaluation in Python. While the templates are still used to generate - code, they are not executed in a separate process, but inside python itself. - """ - assert oracle.language == "python" - _logger.debug("Doing evaluation in Python mode.") - - # Create a configs bundle for the language of the oracle. - eval_bundle = create_bundle( - bundle.config, bundle.out, bundle.suite, oracle.language - ) - - # Path to the oracle. - origin_path = Path(bundle.config.resources, oracle.function.file) - # Read oracle to file. - with open(origin_path, "r") as file: - evaluator_code = file.read() - - # We must provide the globals from the "evaluation_utils" to the code. - # Begin by defining the module. - utils = types.ModuleType("evaluation_utils") - utils.__dict__["EvaluationResult"] = BooleanEvalResult - utils.__dict__["Message"] = ExtendedMessage - - # The context in which to execute. - global_env = {"__tested_test__": utils} - exec("import sys\n" "sys.modules['evaluation_utils'] = __tested_test__", global_env) - # Make the oracle available. - exec(evaluator_code, global_env) - - # Call the oracle. - check_function_call = FunctionCall( - type=FunctionType.FUNCTION, - name=oracle.function.name, - arguments=[expected, actual, *oracle.arguments], - ) - literal_function_call = generate_statement(eval_bundle, check_function_call) - - with _catch_output() as (stdout_, stderr_): - exec(f"__tested_test__result = {literal_function_call}", global_env) - - stdout_ = stdout_.getvalue() - stderr_ = stderr_.getvalue() - - messages = [] - if stdout_: - messages.append( - ExtendedMessage( - description=get_i18n_string("judge.programmed.produced.stdout"), - format="text", - ) - ) - messages.append(ExtendedMessage(description=stdout_, format="code")) - if stderr_: - messages.append( - ExtendedMessage( - description=get_i18n_string("judge.programmed.produced.stderr"), - format="text", - permission=Permission.STUDENT, - ) - ) - messages.append( - ExtendedMessage( - description=stderr_, format="code", permission=Permission.STAFF - ) - ) - - result_ = cast(BooleanEvalResult | None, global_env["__tested_test__result"]) - - # If the result is None, the oracle is broken. - if result_ is None: - messages.append( - ExtendedMessage( - description=get_i18n_string("judge.programmed.student"), format="text" - ) - ) - messages.append( - ExtendedMessage( - description=get_i18n_string("judge.programmed.failed"), - format="text", - permission=Permission.STAFF, - ) - ) - return BooleanEvalResult( - result=Status.INTERNAL_ERROR, - readable_expected=None, - readable_actual=None, - messages=messages, - ) - - result_.messages.extend(messages) - return result_ diff --git a/tested/languages/config.py b/tested/languages/config.py index 0bc5b1b6..1fcd4ff4 100644 --- a/tested/languages/config.py +++ b/tested/languages/config.py @@ -21,7 +21,7 @@ conventionalize_namespace, submission_name, ) -from tested.serialisation import FunctionCall, Statement, Value +from tested.serialisation import Statement, Value if typing.TYPE_CHECKING: from tested.configs import GlobalConfig @@ -289,18 +289,6 @@ def modify_solution(self, solution: Path): """ pass - def modify_specific_evaluator(self, evaluator: Path): - """ - An opportunity to modify the language-specific oracle. By default, - this does nothing. If you modify the oracle, you must overwrite the - contents of the oracle in-place. - - This callback is called before any compilation. - - :param evaluator: Path to the oracle and path for the modified oracle. - """ - pass - def compiler_output( self, stdout: str, stderr: str ) -> tuple[list[Message], list[AnnotateCode], str, str]: @@ -443,16 +431,6 @@ def generate_selector(self, contexts: list[str]) -> str: """ raise NotImplementedError - def generate_check_function(self, name: str, function: FunctionCall) -> str: - """ - Generate code that calls the given function as a custom check function. - - :param name: The name of the custom check function. - :param function: The function to call. - :return: A string representing the custom check function. - """ - raise NotImplementedError - @abstractmethod def generate_encoder(self, values: list[Value]) -> str: """ diff --git a/tested/languages/csharp/config.py b/tested/languages/csharp/config.py index 5d05e607..835e22a0 100644 --- a/tested/languages/csharp/config.py +++ b/tested/languages/csharp/config.py @@ -20,7 +20,7 @@ submission_file, submission_name, ) -from tested.serialisation import FunctionCall, Statement, Value +from tested.serialisation import Statement, Value logger = logging.getLogger(__name__) @@ -233,11 +233,6 @@ def generate_selector(self, contexts: list[str]) -> str: return generators.convert_selector(contexts) - def generate_check_function(self, name: str, function: FunctionCall) -> str: - from tested.languages.csharp import generators - - return generators.convert_check_function(function) - def generate_encoder(self, values: list[Value]) -> str: from tested.languages.csharp import generators diff --git a/tested/languages/generation.py b/tested/languages/generation.py index a5eba6b8..880794bc 100644 --- a/tested/languages/generation.py +++ b/tested/languages/generation.py @@ -10,7 +10,7 @@ from collections.abc import Iterable from pathlib import Path from re import Match -from typing import TypeAlias +from typing import TYPE_CHECKING, TypeAlias from pygments import highlight from pygments.formatters.html import HtmlFormatter @@ -20,33 +20,18 @@ from tested.datatypes import AllTypes, BasicObjectTypes from tested.dodona import ExtendedMessage from tested.internationalization import get_i18n_string -from tested.judge.planning import PlannedExecutionUnit from tested.languages import Language -from tested.languages.conventionalize import ( - conventionalize_namespace, - selector_file, - submission_name, -) +from tested.languages.conventionalize import selector_file, submission_name from tested.languages.preparation import ( PreparedExecutionUnit, - PreparedFunctionCall, prepare_assignment, prepare_execution_unit, prepare_expression, ) from tested.parsing import get_converter -from tested.serialisation import ( - Assignment, - Expression, - FunctionType, - Identifier, - Statement, - Value, - VariableType, -) +from tested.serialisation import Assignment, Expression, Statement, VariableType from tested.testsuite import ( Context, - CustomCheckOracle, FileUrl, LanguageLiterals, MainInput, @@ -54,6 +39,9 @@ TextData, ) +if TYPE_CHECKING: + from tested.judge.planning import PlannedExecutionUnit + _logger = logging.getLogger(__name__) _html_formatter = HtmlFormatter(nowrap=True) @@ -248,7 +236,7 @@ def generate_statement(bundle: Bundle, statement: Statement) -> str: def generate_execution( bundle: Bundle, destination: Path, - execution_unit: PlannedExecutionUnit, + execution_unit: "PlannedExecutionUnit", ) -> tuple[str, list[str]]: """ Generate the files related to the execution. @@ -298,47 +286,6 @@ def generate_selector( return selector_filename -def generate_custom_evaluator( - bundle: Bundle, - destination: Path, - evaluator: CustomCheckOracle, - expected_value: Value, - actual_value: Value, -) -> str: - """ - Generate the code for running a programmed oracle. - - :param bundle: The configuration bundle. - :param destination: The folder where the code should be generated. - :param evaluator: The oracle data from the test suite. - :param expected_value: The preprocessed expected value. - :param actual_value: The preprocessed actual value. - - :return: The name of the generated file. - """ - evaluator_name = conventionalize_namespace( - bundle.language, evaluator.function.file.stem - ) - - function = PreparedFunctionCall( - type=FunctionType.FUNCTION, - namespace=Identifier(evaluator_name), - name=evaluator.function.name, - arguments=[expected_value, actual_value, *evaluator.arguments], - has_root_namespace=False, - ) - - code = bundle.language.generate_check_function(evaluator_name, function) - - if destination.is_dir(): - destination /= bundle.language.with_extension("EvaluatorExecutor") - - with open(destination, "w") as check_function_file: - check_function_file.write(code) - - return destination.name - - def _convert_single_type( language: Language, type_: AllTypes | VariableType, inner: bool ) -> str | bool: diff --git a/tested/languages/haskell/config.py b/tested/languages/haskell/config.py index ae0ad4bf..da8c523b 100644 --- a/tested/languages/haskell/config.py +++ b/tested/languages/haskell/config.py @@ -21,7 +21,7 @@ executable_name, haskell_solution, ) -from tested.serialisation import FunctionCall, Statement, Value +from tested.serialisation import Statement, Value if TYPE_CHECKING: from tested.languages.generation import PreparedExecutionUnit @@ -175,11 +175,6 @@ def generate_selector(self, contexts: list[str]) -> str: return generators.convert_selector(contexts) - def generate_check_function(self, name: str, function: FunctionCall) -> str: - from tested.languages.haskell import generators - - return generators.convert_check_function(name, function) - def generate_encoder(self, values: list[Value]) -> str: from tested.languages.haskell import generators diff --git a/tested/languages/java/config.py b/tested/languages/java/config.py index 5b38e262..a6ea4b70 100644 --- a/tested/languages/java/config.py +++ b/tested/languages/java/config.py @@ -18,7 +18,7 @@ submission_file, ) from tested.languages.utils import jvm_cleanup_stacktrace, jvm_memory_limit -from tested.serialisation import FunctionCall, Statement, Value +from tested.serialisation import Statement, Value if TYPE_CHECKING: from tested.languages.generation import PreparedExecutionUnit @@ -153,11 +153,6 @@ def generate_selector(self, contexts: list[str]) -> str: return generators.convert_selector(contexts) - def generate_check_function(self, name: str, function: FunctionCall) -> str: - from tested.languages.java import generators - - return generators.convert_check_function(function) - def generate_encoder(self, values: list[Value]) -> str: from tested.languages.java import generators diff --git a/tested/languages/javascript/config.py b/tested/languages/javascript/config.py index 065753d9..e09f330d 100644 --- a/tested/languages/javascript/config.py +++ b/tested/languages/javascript/config.py @@ -20,7 +20,7 @@ submission_name, ) from tested.languages.utils import cleanup_description -from tested.serialisation import FunctionCall, Statement, Value +from tested.serialisation import Statement, Value if TYPE_CHECKING: from tested.languages.generation import PreparedExecutionUnit @@ -197,11 +197,6 @@ def generate_execution_unit(self, execution_unit: "PreparedExecutionUnit") -> st return generators.convert_execution_unit(execution_unit) - def generate_check_function(self, name: str, function: FunctionCall) -> str: - from tested.languages.javascript import generators - - return generators.convert_check_function(name, function) - def generate_encoder(self, values: list[Value]) -> str: from tested.languages.javascript import generators diff --git a/tested/languages/kotlin/config.py b/tested/languages/kotlin/config.py index 1986794e..80f5466a 100644 --- a/tested/languages/kotlin/config.py +++ b/tested/languages/kotlin/config.py @@ -21,7 +21,7 @@ submission_file, ) from tested.languages.utils import jvm_cleanup_stacktrace, jvm_memory_limit -from tested.serialisation import FunctionCall, Statement, Value +from tested.serialisation import Statement, Value if TYPE_CHECKING: from tested.languages.generation import PreparedExecutionUnit @@ -223,11 +223,6 @@ def generate_selector(self, contexts: list[str]) -> str: return generators.convert_selector(contexts) - def generate_check_function(self, name: str, function: FunctionCall) -> str: - from tested.languages.kotlin import generators - - return generators.convert_check_function(function) - def generate_encoder(self, values: list[Value]) -> str: from tested.languages.kotlin import generators diff --git a/tested/languages/python/config.py b/tested/languages/python/config.py index aa7b0928..6427607d 100644 --- a/tested/languages/python/config.py +++ b/tested/languages/python/config.py @@ -18,7 +18,7 @@ NamingConventions, submission_file, ) -from tested.serialisation import FunctionCall, Statement, Value +from tested.serialisation import Statement, Value if TYPE_CHECKING: from tested.languages.generation import PreparedExecutionUnit @@ -223,11 +223,6 @@ def generate_execution_unit(self, execution_unit: "PreparedExecutionUnit") -> st return generators.convert_execution_unit(execution_unit) - def generate_check_function(self, name: str, function: FunctionCall) -> str: - from tested.languages.python import generators - - return generators.convert_check_function(name, function) - def generate_encoder(self, values: list[Value]) -> str: from tested.languages.python import generators diff --git a/tested/manual.py b/tested/manual.py index 6063a0b2..fef7da69 100644 --- a/tested/manual.py +++ b/tested/manual.py @@ -12,7 +12,7 @@ from tested.main import run from tested.testsuite import SupportedLanguage -exercise_dir = "/home/niko/Ontwikkeling/universal-judge/tests/exercises/echo" +exercise_dir = "/home/niko/Ontwikkeling/universal-judge/tests/exercises/echo-function" def read_config() -> DodonaConfig: @@ -20,13 +20,13 @@ def read_config() -> DodonaConfig: return DodonaConfig( memory_limit=536870912, time_limit=60, - programming_language=SupportedLanguage("python"), + programming_language=SupportedLanguage("haskell"), natural_language="nl", resources=Path(exercise_dir, "evaluation"), - source=Path(exercise_dir, "solution/correct.py"), + source=Path(exercise_dir, "solution/correct.hs"), judge=Path("."), workdir=Path("workdir"), - test_suite="one.tson", + test_suite="two-specific.tson", options=Options( linter=False, ), diff --git a/tested/oracles/common.py b/tested/oracles/common.py index 0859fb2a..3e4a6068 100644 --- a/tested/oracles/common.py +++ b/tested/oracles/common.py @@ -35,9 +35,20 @@ def evaluate_text(configs, channel, actual): from tested.languages.generation import generate_statement from tested.languages.utils import convert_stacktrace_to_clickable_feedback from tested.parsing import fallback_field, get_converter +from tested.serialisation import Value from tested.testsuite import ExceptionOutputChannel, NormalOutputChannel, OutputChannel +@define +class OracleContext: + expected: Value + actual: Value + execution_directory: str + evaluation_directory: str + programming_language: str + natural_language: str + + @define class OracleResult: """ diff --git a/tested/oracles/programmed.py b/tested/oracles/programmed.py index 10be88f6..bee5e0e3 100644 --- a/tested/oracles/programmed.py +++ b/tested/oracles/programmed.py @@ -1,13 +1,28 @@ +import contextlib import logging +import sys import traceback +from io import StringIO +from pathlib import Path +from types import ModuleType +from typing import Any, Generator, cast +from attrs import define + +from tested.configs import Bundle, create_bundle from tested.dodona import ExtendedMessage, Message, Permission, Status, StatusMessage from tested.internationalization import get_i18n_string -from tested.judge.programmed import evaluate_programmed from tested.judge.utils import BaseExecutionResult -from tested.oracles.common import BooleanEvalResult, OracleConfig, OracleResult +from tested.languages.generation import generate_statement +from tested.oracles.common import ( + BooleanEvalResult, + OracleConfig, + OracleContext, + OracleResult, +) from tested.oracles.value import get_values from tested.parsing import get_converter +from tested.serialisation import FunctionCall, FunctionType, Identifier from tested.testsuite import CustomCheckOracle, OracleOutputChannel, OutputChannel _logger = logging.getLogger(__name__) @@ -15,6 +30,145 @@ DEFAULT_STUDENT = get_i18n_string("oracles.programmed.student.default") +@define +class ConvertedOracleContext: + expected: Any + actual: Any + execution_directory: str + evaluation_directory: str + programming_language: str + natural_language: str + + @staticmethod + def from_context( + bundle: Bundle, context: OracleContext + ) -> "ConvertedOracleContext": + return ConvertedOracleContext( + expected=eval(generate_statement(bundle, context.expected)), + actual=eval(generate_statement(bundle, context.actual)), + execution_directory=context.execution_directory, + evaluation_directory=context.evaluation_directory, + programming_language=context.programming_language, + natural_language=context.natural_language, + ) + + +@contextlib.contextmanager +def _catch_output() -> Generator[tuple[StringIO, StringIO], None, None]: + old_stdout = sys.stdout + old_stderr = sys.stderr + stdout = StringIO() + stderr = StringIO() + try: + sys.stdout = stdout + sys.stderr = stderr + yield stdout, stderr + finally: + sys.stdout = old_stdout + sys.stderr = old_stderr + + +def _evaluate_programmed( + bundle: Bundle, + oracle: CustomCheckOracle, + context: OracleContext, +) -> BaseExecutionResult | BooleanEvalResult: + """ + Run the custom evaluation. Concerning structure and execution, the custom + oracle is very similar to the execution of the whole evaluation. It a + mini-evaluation if you will. + """ + _logger.debug("Doing evaluation in Python mode.") + + # Create a configs bundle for the language of the oracle. + eval_bundle = create_bundle(bundle.config, bundle.out, bundle.suite, "python") + + # Path to the oracle. + origin_path = Path(bundle.config.resources, oracle.function.file) + # Read oracle to file. + with open(origin_path, "r") as file: + evaluator_code = file.read() + + # We must provide the globals from the "evaluation_utils" to the code. + # Begin by defining the module. + utils = ModuleType("evaluation_utils") + utils.__dict__["EvaluationResult"] = BooleanEvalResult + utils.__dict__["Message"] = ExtendedMessage + utils.__dict__["ConvertedOracleContext"] = ConvertedOracleContext + + # The context in which to execute. + global_env = { + "__tested_test__": utils, + "__tested_context__": ConvertedOracleContext.from_context(eval_bundle, context), + } + exec("import sys\n" "sys.modules['evaluation_utils'] = __tested_test__", global_env) + # Make the oracle available. + exec(evaluator_code, global_env) + + # Since we pass a class value, we don't want to + check_function_call = FunctionCall( + type=FunctionType.FUNCTION, + name=oracle.function.name, + arguments=[Identifier("__tested_context__"), *oracle.arguments], + ) + literal_function_call = generate_statement(eval_bundle, check_function_call) + + with _catch_output() as (stdout_, stderr_): + exec(f"__tested_test__result = {literal_function_call}", global_env) + + stdout_ = stdout_.getvalue() + stderr_ = stderr_.getvalue() + + messages = [] + if stdout_: + messages.append( + ExtendedMessage( + description=get_i18n_string("judge.programmed.produced.stdout"), + format="text", + ) + ) + messages.append(ExtendedMessage(description=stdout_, format="code")) + if stderr_: + messages.append( + ExtendedMessage( + description=get_i18n_string("judge.programmed.produced.stderr"), + format="text", + permission=Permission.STUDENT, + ) + ) + messages.append( + ExtendedMessage( + description=stderr_, format="code", permission=Permission.STAFF + ) + ) + + result_ = cast(BooleanEvalResult | None, global_env["__tested_test__result"]) + + # If the result is None, the oracle is broken. + if result_ is None: + messages.append( + ExtendedMessage( + description=get_i18n_string("judge.programmed.student"), format="text" + ) + ) + messages.append( + ExtendedMessage( + description=get_i18n_string("judge.programmed.failed"), + format="text", + permission=Permission.STAFF, + ) + ) + return BooleanEvalResult( + result=Status.INTERNAL_ERROR, + readable_expected=None, + readable_actual=None, + messages=messages, + ) + + result_.messages.extend(messages) + return result_ + + def evaluate( config: OracleConfig, channel: OutputChannel, actual_str: str ) -> OracleResult: @@ -51,9 +205,15 @@ def evaluate( f"expected: {expected}\n" f"actual: {actual}" ) - result = evaluate_programmed( - config.bundle, evaluator=channel.oracle, expected=expected, actual=actual + context = OracleContext( + expected=expected, + actual=actual, + execution_directory=str(config.context_dir.absolute()), + evaluation_directory=str(config.bundle.config.resources.absolute()), + programming_language=str(config.bundle.config.programming_language), + natural_language=config.bundle.config.natural_language, ) + result = _evaluate_programmed(config.bundle, channel.oracle, context) if isinstance(result, BaseExecutionResult): _logger.error(result.stderr) diff --git a/tested/testsuite.py b/tested/testsuite.py index 7312c32f..d6489fc4 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -162,7 +162,6 @@ class CustomCheckOracle: or evaluating non-deterministic return values. """ - language: str function: EvaluationFunction arguments: list[Value] = field(factory=list) type: Literal["programmed", "custom_check"] = "custom_check" diff --git a/tests/exercises/echo-function/evaluation/Evaluator.cs b/tests/exercises/echo-function/evaluation/Evaluator.cs index 17f74c73..8dc784d4 100644 --- a/tests/exercises/echo-function/evaluation/Evaluator.cs +++ b/tests/exercises/echo-function/evaluation/Evaluator.cs @@ -7,14 +7,4 @@ public static EvaluationResult Evaluate(Object actual) { var messages = new List() {new Tested.Message("Hallo")}; return new EvaluationResult(correct, "correct", actual != null ? actual.ToString() : "", messages); } - - public static EvaluationResult EvaluateValue(Object expected, Object actual) { - var messages = new List() {new Tested.Message("Hallo")}; - return new EvaluationResult(expected == actual, expected.ToString(), actual != null ? actual.ToString() : "", messages); - } - - public static EvaluationResult EvaluateValueDsl(Object expected, Object actual) { - var messages = new List() {new Tested.Message("Hallo")}; - return new EvaluationResult(expected == actual, null, null, messages, "{5, 5}", "{4, 4}"); - } } diff --git a/tests/exercises/echo-function/evaluation/Evaluator.hs b/tests/exercises/echo-function/evaluation/Evaluator.hs index 586f84aa..e68ccc10 100644 --- a/tests/exercises/echo-function/evaluation/Evaluator.hs +++ b/tests/exercises/echo-function/evaluation/Evaluator.hs @@ -13,14 +13,3 @@ evaluate value = readableActual = Just value, messages = [message "Hallo"] } - - -evaluate_value :: String -> String -> EvaluationResult -evaluate_value expected actual = - let correct = if actual == expected then True else False - in evaluationResult { - result = correct, - readableExpected = Just expected, - readableActual = Just actual, - messages = [message "Hallo"] - } diff --git a/tests/exercises/echo-function/evaluation/Evaluator.java b/tests/exercises/echo-function/evaluation/Evaluator.java index f9e9a0a9..c4197043 100644 --- a/tests/exercises/echo-function/evaluation/Evaluator.java +++ b/tests/exercises/echo-function/evaluation/Evaluator.java @@ -10,20 +10,4 @@ public static EvaluationResult evaluate(Object actual) { .withMessage(new EvaluationResult.Message("Hallo")) .build(); } - - public static EvaluationResult evaluateValue(Object expected, Object actual) { - return EvaluationResult.builder(expected.equals(actual)) - .withReadableExpected(expected.toString()) - .withReadableActual(actual != null ? actual.toString() : "") - .withMessage(new EvaluationResult.Message("Hallo")) - .build(); - } - - public static EvaluationResult evaluateValueDsl(Object expected, Object actual) { - return EvaluationResult.builder(expected.equals(actual)) - .withDslExpected("{5, 5}") - .withDslActual("{4, 4}") - .withMessage(new EvaluationResult.Message("Hallo")) - .build(); - } } diff --git a/tests/exercises/echo-function/evaluation/Evaluator.kt b/tests/exercises/echo-function/evaluation/Evaluator.kt index 0f693aa6..99cd36d6 100644 --- a/tests/exercises/echo-function/evaluation/Evaluator.kt +++ b/tests/exercises/echo-function/evaluation/Evaluator.kt @@ -10,17 +10,17 @@ class Evaluator { } @JvmStatic - fun evaluateValue(expected: Any, actual: Any?): EvaluationResult { - return EvaluationResult.Builder(result = expected == actual, - readableExpected = expected.toString(), - readableActual = actual?.toString() ?: "") + fun evaluateValue(context: Map): EvaluationResult { + return EvaluationResult.Builder(result = context["expected"] == context["actual"], + readableExpected = context["expected"].toString(), + readableActual = context["actual"]?.toString() ?: "") .withMessage(EvaluationResult.Message("Hallo")) .build() } @JvmStatic - fun evaluateValueDsl(expected: Any, actual: Any?): EvaluationResult { - return EvaluationResult.Builder(result = expected == actual, + fun evaluateValueDsl(context: Map): EvaluationResult { + return EvaluationResult.Builder(result = context["expected"] == context["actual"], dslExpected = "{5, 5}", dslActual = "{4, 4}") .withMessage(EvaluationResult.Message("Hallo")) diff --git a/tests/exercises/echo-function/evaluation/evaluator.js b/tests/exercises/echo-function/evaluation/evaluator.js index 0b283517..10f8b7de 100644 --- a/tests/exercises/echo-function/evaluation/evaluator.js +++ b/tests/exercises/echo-function/evaluation/evaluator.js @@ -8,24 +8,4 @@ function evaluate(actual) { } } -function evaluateValue(expected, actual) { - return { - "result": expected === actual, - "readable_expected": expected, - "readable_actual": actual, - "messages": [{"description": "Hallo", "format": "text"}] - } -} - -function evaluateValueDsl(expected, actual) { - return { - "result": expected === actual, - "dsl_expected": "{5, 5}", - "dsl_actual": "{4, 4}", - "messages": [{"description": "Hallo", "format": "text"}] - } -} - exports.evaluate = evaluate; -exports.evaluateValue = evaluateValue; -exports.evaluateValueDsl = evaluateValueDsl; diff --git a/tests/exercises/echo-function/evaluation/evaluator.py b/tests/exercises/echo-function/evaluation/evaluator.py index e4046066..02304728 100644 --- a/tests/exercises/echo-function/evaluation/evaluator.py +++ b/tests/exercises/echo-function/evaluation/evaluator.py @@ -7,13 +7,13 @@ def evaluate(actual): return EvaluationResult(correct, "correct", actual, [Message("Hallo")]) -def evaluate_value(expected, actual): - return EvaluationResult(expected == actual, expected, actual, [Message("Hallo")]) +def evaluate_value(context): + return EvaluationResult(context.expected == context.actual, context.expected, context.actual, [Message("Hallo")]) -def evaluate_value_dsl(expected, actual): +def evaluate_value_dsl(context): return EvaluationResult( - result=expected == actual, + result=context.expected == context.actual, messages=[Message("Hallo")], dsl_expected="{5, 5}", dsl_actual="{4, 4}" diff --git a/tests/exercises/echo-function/evaluation/programmed-dsl-no-haskell.tson b/tests/exercises/echo-function/evaluation/programmed-dsl-no-haskell.tson deleted file mode 100644 index 65a6703f..00000000 --- a/tests/exercises/echo-function/evaluation/programmed-dsl-no-haskell.tson +++ /dev/null @@ -1,157 +0,0 @@ -{ - "tabs": [ - { - "name": "Tab", - "runs": [ - { - "contexts": [ - { - "testcases": [ - { - "input": { - "type": "function", - "name": "echo", - "arguments": [ - { - "type": "text", - "data": "input-2" - } - ] - }, - "output": { - "result": { - "value": { - "type": "text", - "data": "input-2" - }, - "evaluator": { - "type": "programmed", - "function": { - "file": "Evaluator.java", - "name": "evaluateValueDsl" - }, - "language": "java" - } - } - } - }, - { - "input": { - "type": "function", - "name": "echo", - "arguments": [ - { - "type": "text", - "data": "input-3" - } - ] - }, - "output": { - "result": { - "value": { - "type": "text", - "data": "input-3" - }, - "evaluator": { - "type": "programmed", - "function": { - "file": "evaluator.py", - "name": "evaluate_value_dsl" - }, - "language": "python" - } - } - } - }, - { - "input": { - "type": "function", - "name": "echo", - "arguments": [ - { - "type": "text", - "data": "input-4" - } - ] - }, - "output": { - "result": { - "value": { - "type": "text", - "data": "input-4" - }, - "evaluator": { - "type": "programmed", - "function": { - "file": "evaluator.js", - "name": "evaluateValueDsl" - }, - "language": "javascript" - } - } - } - }, - { - "input": { - "type": "function", - "name": "echo", - "arguments": [ - { - "type": "text", - "data": "input-5" - } - ] - }, - "output": { - "result": { - "value": { - "type": "text", - "data": "input-5" - }, - "evaluator": { - "type": "programmed", - "function": { - "file": "Evaluator.kt", - "name": "evaluateValueDsl" - }, - "language": "kotlin" - } - } - } - }, - { - "input": { - "type": "function", - "name": "echo", - "arguments": [ - { - "type": "text", - "data": "input-89" - } - ] - }, - "output": { - "result": { - "value": { - "type": "text", - "data": "input-89" - }, - "evaluator": { - "type": "programmed", - "function": { - "file": "Evaluator.cs", - "name": "EvaluateValueDsl" - }, - "language": "csharp" - } - } - } - } - ] - } - ] - } - ] - } - ] -} diff --git a/tests/exercises/echo-function/evaluation/programmed-no-haskell.tson b/tests/exercises/echo-function/evaluation/programmed-no-haskell.tson deleted file mode 100644 index 42734310..00000000 --- a/tests/exercises/echo-function/evaluation/programmed-no-haskell.tson +++ /dev/null @@ -1,157 +0,0 @@ -{ - "tabs": [ - { - "name": "Tab", - "runs": [ - { - "contexts": [ - { - "testcases": [ - { - "input": { - "type": "function", - "name": "echo", - "arguments": [ - { - "type": "text", - "data": "input-2" - } - ] - }, - "output": { - "result": { - "value": { - "type": "text", - "data": "input-2" - }, - "evaluator": { - "type": "programmed", - "function": { - "file": "Evaluator.java", - "name": "evaluateValue" - }, - "language": "java" - } - } - } - }, - { - "input": { - "type": "function", - "name": "echo", - "arguments": [ - { - "type": "text", - "data": "input-3" - } - ] - }, - "output": { - "result": { - "value": { - "type": "text", - "data": "input-3" - }, - "evaluator": { - "type": "programmed", - "function": { - "file": "evaluator.py", - "name": "evaluate_value" - }, - "language": "python" - } - } - } - }, - { - "input": { - "type": "function", - "name": "echo", - "arguments": [ - { - "type": "text", - "data": "input-4" - } - ] - }, - "output": { - "result": { - "value": { - "type": "text", - "data": "input-4" - }, - "evaluator": { - "type": "programmed", - "function": { - "file": "evaluator.js", - "name": "evaluateValue" - }, - "language": "javascript" - } - } - } - }, - { - "input": { - "type": "function", - "name": "echo", - "arguments": [ - { - "type": "text", - "data": "input-5" - } - ] - }, - "output": { - "result": { - "value": { - "type": "text", - "data": "input-5" - }, - "evaluator": { - "type": "programmed", - "function": { - "file": "Evaluator.kt", - "name": "evaluateValue" - }, - "language": "kotlin" - } - } - } - }, - { - "input": { - "type": "function", - "name": "echo", - "arguments": [ - { - "type": "text", - "data": "input-89" - } - ] - }, - "output": { - "result": { - "value": { - "type": "text", - "data": "input-89" - }, - "evaluator": { - "type": "programmed", - "function": { - "file": "Evaluator.cs", - "name": "EvaluateValue" - }, - "language": "csharp" - } - } - } - } - ] - } - ] - } - ] - } - ] -} diff --git a/tests/exercises/echo-function/evaluation/programmed.tson b/tests/exercises/echo-function/evaluation/programmed.tson index 8aaa063b..292ee72e 100644 --- a/tests/exercises/echo-function/evaluation/programmed.tson +++ b/tests/exercises/echo-function/evaluation/programmed.tson @@ -7,62 +7,6 @@ "contexts": [ { "testcases": [ - { - "input": { - "type": "function", - "name": "echo", - "arguments": [ - { - "type": "text", - "data": "input-1" - } - ] - }, - "output": { - "result": { - "value": { - "type": "text", - "data": "input-1" - }, - "evaluator": { - "type": "programmed", - "function": { - "file": "Evaluator.hs", - "name": "evaluate_value" - }, - "language": "haskell" - } - } - } - }, - { - "input": { - "type": "function", - "name": "echo", - "arguments": [ - { - "type": "text", - "data": "input-2" - } - ] - }, - "output": { - "result": { - "value": { - "type": "text", - "data": "input-2" - }, - "evaluator": { - "type": "programmed", - "function": { - "file": "Evaluator.java", - "name": "evaluateValue" - }, - "language": "java" - } - } - } - }, { "input": { "type": "function", @@ -85,64 +29,7 @@ "function": { "file": "evaluator.py", "name": "evaluate_value" - }, - "language": "python" - } - } - } - }, - { - "input": { - "type": "function", - "name": "echo", - "arguments": [ - { - "type": "text", - "data": "input-4" - } - ] - }, - "output": { - "result": { - "value": { - "type": "text", - "data": "input-4" - }, - "evaluator": { - "type": "programmed", - "function": { - "file": "evaluator.js", - "name": "evaluateValue" - }, - "language": "javascript" - } - } - } - }, - { - "input": { - "type": "function", - "name": "echo", - "arguments": [ - { - "type": "text", - "data": "input-5" - } - ] - }, - "output": { - "result": { - "value": { - "type": "text", - "data": "input-5" - }, - "evaluator": { - "type": "programmed", - "function": { - "file": "Evaluator.kt", - "name": "evaluateValue" - }, - "language": "kotlin" + } } } } diff --git a/tests/exercises/echo/evaluation/evaluator.py b/tests/exercises/echo/evaluation/evaluator.py index 0c99dcb1..e250a978 100644 --- a/tests/exercises/echo/evaluation/evaluator.py +++ b/tests/exercises/echo/evaluation/evaluator.py @@ -2,9 +2,9 @@ from evaluation_utils import EvaluationResult -def evaluate_correct(expected, actual): - return EvaluationResult(expected.strip() == actual.strip()) +def evaluate_correct(context): + return EvaluationResult(context.expected.strip() == context.actual.strip()) -def evaluate_wrong(_expected, _actual): +def evaluate_wrong(_context): return EvaluationResult(False) diff --git a/tests/exercises/echo/evaluation/one-programmed-correct.tson b/tests/exercises/echo/evaluation/one-programmed-correct.tson index 7e19d052..cc733d12 100644 --- a/tests/exercises/echo/evaluation/one-programmed-correct.tson +++ b/tests/exercises/echo/evaluation/one-programmed-correct.tson @@ -18,7 +18,6 @@ "data": "input-1", "evaluator": { "type": "programmed", - "language": "python", "function": { "file": "evaluator.py", "name": "evaluate_correct" diff --git a/tests/exercises/echo/evaluation/one-programmed-wrong.tson b/tests/exercises/echo/evaluation/one-programmed-wrong.tson index f919d08e..fa1fe2a7 100644 --- a/tests/exercises/echo/evaluation/one-programmed-wrong.tson +++ b/tests/exercises/echo/evaluation/one-programmed-wrong.tson @@ -18,7 +18,6 @@ "data": "input-1", "evaluator": { "type": "programmed", - "language": "python", "function": { "file": "evaluator.py", "name": "evaluate_wrong" diff --git a/tests/exercises/lotto/evaluation/evaluator.py b/tests/exercises/lotto/evaluation/evaluator.py index a2c5c87e..335c7ac4 100644 --- a/tests/exercises/lotto/evaluation/evaluator.py +++ b/tests/exercises/lotto/evaluation/evaluator.py @@ -40,7 +40,9 @@ def valid_lottery_numbers(number_str, count=6, maximum=42): return True, None -def evaluate(expected, actual, count, maximum): +def evaluate(context, count, maximum): + expected = context.expected + actual = context.actual valid, message = valid_lottery_numbers(actual, count, maximum) messages = [Message(message)] if message else [] if valid: diff --git a/tests/exercises/lotto/evaluation/one-programmed-python.tson b/tests/exercises/lotto/evaluation/one-programmed-python.tson index 3acb882f..2466a873 100644 --- a/tests/exercises/lotto/evaluation/one-programmed-python.tson +++ b/tests/exercises/lotto/evaluation/one-programmed-python.tson @@ -16,7 +16,6 @@ }, "evaluator": { "type": "programmed", - "language": "python", "function": { "file": "./evaluator.py" }, diff --git a/tests/exercises/lotto/evaluation/plan.tson b/tests/exercises/lotto/evaluation/plan.tson index 09cf7ad5..7b2fb095 100644 --- a/tests/exercises/lotto/evaluation/plan.tson +++ b/tests/exercises/lotto/evaluation/plan.tson @@ -30,7 +30,6 @@ }, "evaluator": { "type": "programmed", - "language": "python", "function": { "file": "./evaluator.py" }, @@ -75,7 +74,6 @@ }, "evaluator": { "type": "programmed", - "language": "python", "function": { "file": "./evaluator.py" }, @@ -120,7 +118,6 @@ }, "evaluator": { "type": "programmed", - "language": "python", "function": { "file": "./evaluator.py" }, @@ -165,7 +162,6 @@ }, "evaluator": { "type": "programmed", - "language": "python", "function": { "file": "./evaluator.py" }, @@ -210,7 +206,6 @@ }, "evaluator": { "type": "programmed", - "language": "python", "function": { "file": "./evaluator.py" }, @@ -255,7 +250,6 @@ }, "evaluator": { "type": "programmed", - "language": "python", "function": { "file": "./evaluator.py" }, @@ -300,7 +294,6 @@ }, "evaluator": { "type": "programmed", - "language": "python", "function": { "file": "./evaluator.py" }, @@ -345,7 +338,6 @@ }, "evaluator": { "type": "programmed", - "language": "python", "function": { "file": "./evaluator.py" }, @@ -390,7 +382,6 @@ }, "evaluator": { "type": "programmed", - "language": "python", "function": { "file": "./evaluator.py" }, @@ -435,7 +426,6 @@ }, "evaluator": { "type": "programmed", - "language": "python", "function": { "file": "./evaluator.py" }, @@ -480,7 +470,6 @@ }, "evaluator": { "type": "programmed", - "language": "python", "function": { "file": "./evaluator.py" }, @@ -525,7 +514,6 @@ }, "evaluator": { "type": "programmed", - "language": "python", "function": { "file": "./evaluator.py" }, @@ -570,7 +558,6 @@ }, "evaluator": { "type": "programmed", - "language": "python", "function": { "file": "./evaluator.py" }, @@ -615,7 +602,6 @@ }, "evaluator": { "type": "programmed", - "language": "python", "function": { "file": "./evaluator.py" }, @@ -660,7 +646,6 @@ }, "evaluator": { "type": "programmed", - "language": "python", "function": { "file": "./evaluator.py" }, @@ -705,7 +690,6 @@ }, "evaluator": { "type": "programmed", - "language": "python", "function": { "file": "./evaluator.py" }, @@ -750,7 +734,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -795,7 +779,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -840,7 +824,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -885,7 +869,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -930,7 +914,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -975,7 +959,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -1020,7 +1004,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -1065,7 +1049,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -1110,7 +1094,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -1155,7 +1139,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -1200,7 +1184,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -1245,7 +1229,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -1290,7 +1274,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -1335,7 +1319,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -1380,7 +1364,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -1425,7 +1409,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -1470,7 +1454,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -1515,7 +1499,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -1560,7 +1544,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -1605,7 +1589,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -1650,7 +1634,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -1695,7 +1679,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -1740,7 +1724,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -1785,7 +1769,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -1830,7 +1814,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -1875,7 +1859,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -1920,7 +1904,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -1965,7 +1949,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -2010,7 +1994,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, diff --git a/tests/exercises/lotto/evaluation/two.tson b/tests/exercises/lotto/evaluation/two.tson index a3e5f404..11434ace 100644 --- a/tests/exercises/lotto/evaluation/two.tson +++ b/tests/exercises/lotto/evaluation/two.tson @@ -16,7 +16,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, @@ -61,7 +61,7 @@ }, "evaluator": { "type": "programmed", - "language": "python", + "function": { "file": "./evaluator.py" }, diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index d9b7ebef..12155eec 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -595,7 +595,6 @@ def test_text_custom_checks_correct(): stdout: data: "hallo" oracle: "custom_check" - language: "python" file: "test.py" name: "evaluate_test" arguments: [!expression "'yes'", 5, !expression "set([5, 5])"] @@ -613,7 +612,6 @@ def test_text_custom_checks_correct(): assert isinstance(test.output.stdout.oracle, CustomCheckOracle) assert test.output.stdout.data == "hallo" oracle = test.output.stdout.oracle - assert oracle.language == "python" assert oracle.function.name == "evaluate_test" assert oracle.function.file == Path("test.py") assert oracle.arguments == [ @@ -689,7 +687,6 @@ def test_value_custom_checks_correct(): return: !oracle value: "hallo" oracle: "custom_check" - language: "python" file: "test.py" name: "evaluate_test" arguments: ['yes', 5, !expression "set([5, 5])"] @@ -709,7 +706,6 @@ def test_value_custom_checks_correct(): type=BasicStringTypes.TEXT, data="hallo" ) oracle = test.output.result.oracle - assert oracle.language == "python" assert oracle.function.name == "evaluate_test" assert oracle.function.file == Path("test.py") assert oracle.arguments == [ diff --git a/tests/test_functionality.py b/tests/test_functionality.py index 284bc045..a9f73522 100644 --- a/tests/test_functionality.py +++ b/tests/test_functionality.py @@ -284,32 +284,13 @@ def test_programmed_evaluation(language: str, tmp_path: Path, pytestconfig): "echo-function", language, tmp_path, - "programmed-no-haskell.tson", + "programmed.tson", "correct", ) result = execute_config(conf) updates = assert_valid_output(result, pytestconfig) - assert updates.find_status_enum() == ["correct"] * 5 - assert len(updates.find_all("append-message")) == 5 - - -def test_programmed_evaluation_with_dsl(tmp_path: Path, pytestconfig): - conf = configuration( - pytestconfig, - "echo-function", - "javascript", - tmp_path, - "programmed-dsl-no-haskell.tson", - "correct", - ) - result = execute_config(conf) - updates = assert_valid_output(result, pytestconfig) - assert updates.find_status_enum() == ["correct"] * 5 - assert len(updates.find_all("append-message")) == 5 - all_expected = [x["expected"] for x in updates.find_all("start-test")] - assert all_expected == ["new Set([5, 5])"] * 5 - all_actual = [x["generated"] for x in updates.find_all("close-test")] - assert all_actual == ["new Set([4, 4])"] * 5 + assert updates.find_status_enum() == ["correct"] + assert len(updates.find_all("append-message")) @pytest.mark.parametrize( diff --git a/tests/test_suite.py b/tests/test_suite.py index 4f22b8f6..7c61d485 100644 --- a/tests/test_suite.py +++ b/tests/test_suite.py @@ -13,7 +13,6 @@ def test_text_output_is_compatible_oracle(): old_structure = { "evaluator": { - "language": "python", "function": {"file": "evaluate.py"}, "type": "custom_check", }, @@ -23,14 +22,12 @@ def test_text_output_is_compatible_oracle(): result = get_converter().structure(old_structure, TextOutputChannel) assert isinstance(result.oracle, CustomCheckOracle) - assert result.oracle.language == "python" assert result.oracle.function.file.name == "evaluate.py" def test_file_output_is_compatible_oracle(): old_structure = { "evaluator": { - "language": "python", "function": {"file": "evaluate.py"}, "type": "custom_check", }, @@ -41,14 +38,12 @@ def test_file_output_is_compatible_oracle(): result = get_converter().structure(old_structure, FileOutputChannel) assert isinstance(result.oracle, CustomCheckOracle) - assert result.oracle.language == "python" assert result.oracle.function.file.name == "evaluate.py" def test_value_output_is_compatible_oracle(): old_structure = { "evaluator": { - "language": "python", "function": {"file": "evaluate.py"}, "type": "custom_check", }, @@ -58,14 +53,12 @@ def test_value_output_is_compatible_oracle(): result = get_converter().structure(old_structure, ValueOutputChannel) assert isinstance(result.oracle, CustomCheckOracle) - assert result.oracle.language == "python" assert result.oracle.function.file.name == "evaluate.py" def test_exception_output_is_compatible_oracle(): old_structure = { "evaluator": { - "language": "python", "function": {"file": "evaluate.py"}, "type": "custom_check", }, @@ -75,7 +68,6 @@ def test_exception_output_is_compatible_oracle(): result = get_converter().structure(old_structure, ValueOutputChannel) assert isinstance(result.oracle, CustomCheckOracle) - assert result.oracle.language == "python" assert result.oracle.function.file.name == "evaluate.py"