Fix bug where output was not caught when importing

tested/judge/evaluation.py

@@ -197,9 +197,9 @@ def evaluate_context_results(
     values = exec_results.results.split(exec_results.separator)
 
     # The first item should always be empty, since the separator must be printed
-    # before the test suite runs. We remove the first item; for stdout and stderr
-    # we only remove the first item if it is indeed empty. This is to keep error
-    # messages present for debugging.
+    # before the test suite runs. We remove the first item; but only
+    # if it is indeed empty. This is to keep error messages present for
+    # debugging.
 
     deletions = (
         safe_del(stdout_, 0, lambda e: e == ""),

tested/languages/bash/generators.py

@@ -201,19 +201,21 @@ def convert_execution_unit(pu: PreparedExecutionUnit) -> str:
 
         # Import the submission if there is no main call.
         if not ctx.context.has_main_testcase():
+            result += indent + "write_separator\n"
             result += f"{indent}source {submission_file(pu.language)}\n"
 
         # Generate code for each testcase
         tc: PreparedTestcase
-        for tc in ctx.testcases:
-            result += indent + "write_separator\n"
-
+        for j, tc in enumerate(ctx.testcases):
             # Prepare command arguments if needed.
             if tc.testcase.is_main_testcase():
+                result += indent + "write_separator\n"
                 assert isinstance(tc.input, MainInput)
                 result += f"{indent}bash {submission_file(pu.language)} "
                 result += " ".join(shlex.quote(s) for s in tc.input.arguments) + "\n"
             else:
+                if j != 0:
+                    result += indent + "write_separator\n"
                 assert isinstance(tc.input, PreparedTestcaseStatement)
                 result += indent + convert_statement(tc.input.input_statement()) + "\n"
             result += "\n"

tested/languages/javascript/generators.py

@@ -133,24 +133,26 @@ def _generate_internal_context(ctx: PreparedContext, pu: PreparedExecutionUnit)
     # Import the submission if there is no main call.
     if not ctx.context.has_main_testcase():
         result += f"""
+            writeSeparator();
             delete require.cache[require.resolve("./{submission_file(pu.language)}")];
             const {pu.submission_name} = require("./{submission_file(pu.language)}");
             """
 
     # Generate code for each testcase
     tc: PreparedTestcase
-    for tc in ctx.testcases:
-        result += "writeSeparator();\n"
-
+    for i, tc in enumerate(ctx.testcases):
         # Prepare command arguments if needed.
         if tc.testcase.is_main_testcase():
             assert isinstance(tc.input, MainInput)
             wrapped = [json.dumps(a) for a in tc.input.arguments]
             result += f"""
+            writeSeparator();
             let new_args = [process.argv[0]];
             new_args = new_args.concat([{", ".join(wrapped)}]);
             process.argv = new_args;
             """
+        elif i != 0:
+            result += "writeSeparator();\n"
 
         # We need special code to make variables available outside of the try-catch block.
         if (

tested/languages/python/generators.py

@@ -202,23 +202,25 @@ def send_specific_exception(exception):
         result += indent + ctx.before + "\n"
 
         if not ctx.context.has_main_testcase():
+            result += indent + "write_separator()\n"
             result += indent + f"import {pu.submission_name}\n"
             if i != 0:
                 result += (
                     f'{indent}importlib.reload(sys.modules["{pu.submission_name}"])\n'
                 )
 
         tc: PreparedTestcase
-        for tc in ctx.testcases:
-            result += indent + "write_separator()\n"
-
+        for j, tc in enumerate(ctx.testcases):
             # Prepare command arguments if needed.
             if tc.testcase.is_main_testcase():
+                result += indent + "write_separator()\n"
                 assert isinstance(tc.input, MainInput)
                 result += indent + "new_args = [sys.argv[0]]\n"
                 wrapped = [json.dumps(a) for a in tc.input.arguments]
                 result += f"{indent}new_args.extend([{', '.join(wrapped)}])\n"
                 result += indent + "sys.argv = new_args\n"
+            elif j != 0:
+                result += indent + "write_separator()\n"
 
             result += indent + "try:\n"
             if tc.testcase.is_main_testcase():

tests/exercises/echo-function/evaluation/two.yaml

@@ -0,0 +1,8 @@
+- tab: "My tab"
+  contexts:
+    - testcases:
+        - expression: 'echo("input-1")'
+          return: "input-1"
+        - expression: 'echo("input-2")'
+          return: "input-2"
+

tests/exercises/echo-function/solution/top-level-output.js

@@ -0,0 +1,15 @@
+function echo(content) {
+    return content;
+}
+
+
+function noEcho(content) {
+    // Do nothing.
+}
+
+function toString(number) {
+    return number.toString()
+}
+
+
+console.log("This is top-level output");

tests/exercises/echo-function/solution/top-level-output.py

@@ -0,0 +1,13 @@
+def echo(content):
+    return content
+
+
+def no_echo(content):
+    pass
+
+
+def to_string(n):
+    return str(n)
+
+
+print("Hallo?")

tests/exercises/echo-function/solution/top-level-output.sh

@@ -0,0 +1,10 @@
+function my_echo {
+    echo "$1"
+}
+
+
+function no_echo {
+    : # Do nothing here
+}
+
+echo "Hallo, this is output."

tests/test_functionality.py

@@ -982,3 +982,36 @@ def test_python_input_prompt_is_ignored(tmp_path: Path, pytestconfig):
     result = execute_config(conf)
     updates = assert_valid_output(result, pytestconfig)
     assert updates.find_status_enum() == ["correct"]
+
+
+# Check that the test suite is valid with a correct submission.
+# This test suite is used for the test below "test_output_in_script_is_caught".
+@pytest.mark.parametrize("language", ["python", "javascript", "bash"])
+def test_two_suite_is_valid(language: str, tmp_path: Path, pytestconfig):
+    conf = configuration(
+        pytestconfig,
+        "echo-function",
+        "python",
+        tmp_path,
+        "two.yaml",
+        "correct",
+    )
+    result = execute_config(conf)
+    updates = assert_valid_output(result, pytestconfig)
+    assert updates.find_status_enum() == ["correct"] * 2
+
+
+@pytest.mark.parametrize("language", ["python", "javascript", "bash"])
+def test_output_in_script_is_caught(language: str, tmp_path: Path, pytestconfig):
+    conf = configuration(
+        pytestconfig,
+        "echo-function",
+        language,
+        tmp_path,
+        "two.yaml",
+        "top-level-output",
+    )
+    result = execute_config(conf)
+    print(result)
+    updates = assert_valid_output(result, pytestconfig)
+    assert updates.find_status_enum() == ["wrong", "correct", "correct"]