diff --git a/tested/judge/evaluation.py b/tested/judge/evaluation.py index 43502a4d..44b2571a 100644 --- a/tested/judge/evaluation.py +++ b/tested/judge/evaluation.py @@ -197,9 +197,9 @@ def evaluate_context_results( values = exec_results.results.split(exec_results.separator) # The first item should always be empty, since the separator must be printed - # before the test suite runs. We remove the first item; for stdout and stderr - # we only remove the first item if it is indeed empty. This is to keep error - # messages present for debugging. + # before the test suite runs. We remove the first item; but only + # if it is indeed empty. This is to keep error messages present for + # debugging. deletions = ( safe_del(stdout_, 0, lambda e: e == ""), diff --git a/tested/languages/bash/generators.py b/tested/languages/bash/generators.py index a1dfb5f0..e673ef42 100644 --- a/tested/languages/bash/generators.py +++ b/tested/languages/bash/generators.py @@ -201,19 +201,21 @@ def convert_execution_unit(pu: PreparedExecutionUnit) -> str: # Import the submission if there is no main call. if not ctx.context.has_main_testcase(): + result += indent + "write_separator\n" result += f"{indent}source {submission_file(pu.language)}\n" # Generate code for each testcase tc: PreparedTestcase - for tc in ctx.testcases: - result += indent + "write_separator\n" - + for j, tc in enumerate(ctx.testcases): # Prepare command arguments if needed. if tc.testcase.is_main_testcase(): + result += indent + "write_separator\n" assert isinstance(tc.input, MainInput) result += f"{indent}bash {submission_file(pu.language)} " result += " ".join(shlex.quote(s) for s in tc.input.arguments) + "\n" else: + if j != 0: + result += indent + "write_separator\n" assert isinstance(tc.input, PreparedTestcaseStatement) result += indent + convert_statement(tc.input.input_statement()) + "\n" result += "\n" diff --git a/tested/languages/javascript/generators.py b/tested/languages/javascript/generators.py index 2b44856d..2993cf85 100644 --- a/tested/languages/javascript/generators.py +++ b/tested/languages/javascript/generators.py @@ -133,24 +133,26 @@ def _generate_internal_context(ctx: PreparedContext, pu: PreparedExecutionUnit) # Import the submission if there is no main call. if not ctx.context.has_main_testcase(): result += f""" + writeSeparator(); delete require.cache[require.resolve("./{submission_file(pu.language)}")]; const {pu.submission_name} = require("./{submission_file(pu.language)}"); """ # Generate code for each testcase tc: PreparedTestcase - for tc in ctx.testcases: - result += "writeSeparator();\n" - + for i, tc in enumerate(ctx.testcases): # Prepare command arguments if needed. if tc.testcase.is_main_testcase(): assert isinstance(tc.input, MainInput) wrapped = [json.dumps(a) for a in tc.input.arguments] result += f""" + writeSeparator(); let new_args = [process.argv[0]]; new_args = new_args.concat([{", ".join(wrapped)}]); process.argv = new_args; """ + elif i != 0: + result += "writeSeparator();\n" # We need special code to make variables available outside of the try-catch block. if ( diff --git a/tested/languages/python/generators.py b/tested/languages/python/generators.py index 450c7e45..277c8057 100644 --- a/tested/languages/python/generators.py +++ b/tested/languages/python/generators.py @@ -202,6 +202,7 @@ def send_specific_exception(exception): result += indent + ctx.before + "\n" if not ctx.context.has_main_testcase(): + result += indent + "write_separator()\n" result += indent + f"import {pu.submission_name}\n" if i != 0: result += ( @@ -209,16 +210,17 @@ def send_specific_exception(exception): ) tc: PreparedTestcase - for tc in ctx.testcases: - result += indent + "write_separator()\n" - + for j, tc in enumerate(ctx.testcases): # Prepare command arguments if needed. if tc.testcase.is_main_testcase(): + result += indent + "write_separator()\n" assert isinstance(tc.input, MainInput) result += indent + "new_args = [sys.argv[0]]\n" wrapped = [json.dumps(a) for a in tc.input.arguments] result += f"{indent}new_args.extend([{', '.join(wrapped)}])\n" result += indent + "sys.argv = new_args\n" + elif j != 0: + result += indent + "write_separator()\n" result += indent + "try:\n" if tc.testcase.is_main_testcase(): diff --git a/tests/exercises/echo-function/evaluation/two.yaml b/tests/exercises/echo-function/evaluation/two.yaml new file mode 100644 index 00000000..2121a57e --- /dev/null +++ b/tests/exercises/echo-function/evaluation/two.yaml @@ -0,0 +1,8 @@ +- tab: "My tab" + contexts: + - testcases: + - expression: 'echo("input-1")' + return: "input-1" + - expression: 'echo("input-2")' + return: "input-2" + diff --git a/tests/exercises/echo-function/solution/top-level-output.js b/tests/exercises/echo-function/solution/top-level-output.js new file mode 100644 index 00000000..9e89e749 --- /dev/null +++ b/tests/exercises/echo-function/solution/top-level-output.js @@ -0,0 +1,15 @@ +function echo(content) { + return content; +} + + +function noEcho(content) { + // Do nothing. +} + +function toString(number) { + return number.toString() +} + + +console.log("This is top-level output"); diff --git a/tests/exercises/echo-function/solution/top-level-output.py b/tests/exercises/echo-function/solution/top-level-output.py new file mode 100644 index 00000000..3573eb12 --- /dev/null +++ b/tests/exercises/echo-function/solution/top-level-output.py @@ -0,0 +1,13 @@ +def echo(content): + return content + + +def no_echo(content): + pass + + +def to_string(n): + return str(n) + + +print("Hallo?") diff --git a/tests/exercises/echo-function/solution/top-level-output.sh b/tests/exercises/echo-function/solution/top-level-output.sh new file mode 100644 index 00000000..882c1f0b --- /dev/null +++ b/tests/exercises/echo-function/solution/top-level-output.sh @@ -0,0 +1,10 @@ +function my_echo { + echo "$1" +} + + +function no_echo { + : # Do nothing here +} + +echo "Hallo, this is output." diff --git a/tests/test_functionality.py b/tests/test_functionality.py index a9f73522..31faae5c 100644 --- a/tests/test_functionality.py +++ b/tests/test_functionality.py @@ -982,3 +982,36 @@ def test_python_input_prompt_is_ignored(tmp_path: Path, pytestconfig): result = execute_config(conf) updates = assert_valid_output(result, pytestconfig) assert updates.find_status_enum() == ["correct"] + + +# Check that the test suite is valid with a correct submission. +# This test suite is used for the test below "test_output_in_script_is_caught". +@pytest.mark.parametrize("language", ["python", "javascript", "bash"]) +def test_two_suite_is_valid(language: str, tmp_path: Path, pytestconfig): + conf = configuration( + pytestconfig, + "echo-function", + "python", + tmp_path, + "two.yaml", + "correct", + ) + result = execute_config(conf) + updates = assert_valid_output(result, pytestconfig) + assert updates.find_status_enum() == ["correct"] * 2 + + +@pytest.mark.parametrize("language", ["python", "javascript", "bash"]) +def test_output_in_script_is_caught(language: str, tmp_path: Path, pytestconfig): + conf = configuration( + pytestconfig, + "echo-function", + language, + tmp_path, + "two.yaml", + "top-level-output", + ) + result = execute_config(conf) + print(result) + updates = assert_valid_output(result, pytestconfig) + assert updates.find_status_enum() == ["wrong", "correct", "correct"]