Add integration tests

.github/dodona-image-integration.dockerfile

@@ -0,0 +1,12 @@
+# Inherit from the Docker image for Dodona.
+FROM dodona/dodona-tested
+
+# Go back to being root.
+USER root
+WORKDIR /
+
+# Install some additional dependencies needed for testing.
+RUN pip install --no-cache-dir --upgrade pytest pytest-mock pytest-xdist jinja2 marko
+
+# The source of the judge is available in TESTED_SOURCE.
+CMD pytest -n auto ${TESTED_SOURCE}/tests/test_integration_javascript.py

.github/workflows/javascript.yml

@@ -0,0 +1,18 @@
+name: JavaScript integration tests
+
+on: [ workflow_dispatch ]
+
+jobs:
+  # Runs all our JavaScript exercises against TESTed to check if they are still correct.
+  # This is also done in the Docker, since this is hard to do in Nix.
+  javascript-integration:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: git clone git@github.ugent.be:Scriptingtalen/javascript-oefeningen.git
+      - run: git checkout 4a2094135abe972eb38072be129171f685c18ec3
+        working-directory: ./javascript-oefeningen
+      - run: docker build -t "integration-image" -f ${{ github.workspace }}/.github/dodona-image-integration.dockerfile --network=host .
+        name: Build Dodona Docker image
+      - run: docker run -v ${{ github.workspace }}:/github/workspace -e TESTED_SOURCE=/github/workspace -e EXERCISE_REPO=/github/workspace/javascript-oefeningen integration-image
+        name: Run integration tests in Dodona Docker image

flake.nix

@@ -94,17 +94,6 @@
           inherit (python-base-env) projectDir python overrides;
           propagatedBuildInputs = all-other-dependencies;
         };
-
-        unit-test = pkgs.writeShellApplication {
-          name = "unit-test";
-          runtimeInputs = [ python-dev-env pkgs.poetry ] ++ all-other-dependencies;
-          text = ''
-            DOTNET_CLI_HOME="$(mktemp -d)"
-            export DOTNET_CLI_HOME
-            poetry run pytest -n auto --cov=tested --cov-report=xml tests/
-          '';
-        };
-
       in {
         checks = rec {
           default = simple-tests;

pyproject.toml

@@ -15,6 +15,7 @@ markers = [
     "linter", # Run linter tests
     "slow", # Slow tests
 ]
+addopts = "--ignore=tests/test_integration_javascript.py"
 
 [tool.poetry]
 name = "tested"

tests/manual_utils.py

@@ -1,4 +1,5 @@
 import json
+import shutil
 from io import StringIO
 from pathlib import Path
 
@@ -34,28 +35,47 @@ def configuration(
     suite: str = "plan.json",
     solution: str = "solution",
     options=None,
+) -> DodonaConfig:
+    exercise_dir = Path(config.rootdir) / "tests" / "exercises"
+    ep = exercise_dir / exercise
+    return exercise_configuration(
+        config, ep, language, work_dir, suite, solution, options
+    )
+
+
+def exercise_configuration(
+    config,
+    exercise_directory: Path,
+    language: str,
+    work_dir: Path,
+    suite: str,
+    solution: str,
+    options=None,
 ) -> DodonaConfig:
     # Get the file extension for this language.
     ext = get_language(None, language).file_extension()
     if options is None:
         options = {}
-    exercise_dir = Path(config.rootdir) / "tests" / "exercises"
-    ep = f"{exercise_dir}/{exercise}"
     option_dict = recursive_dict_merge(
         {
             "memory_limit": 536870912,
             "time_limit": 3600,  # One hour
             "programming_language": language,
             "natural_language": "nl",
-            "resources": Path(f"{ep}/evaluation"),
-            "source": Path(f"{ep}/solution/{solution}.{ext}"),
+            "resources": exercise_directory / "evaluation",
+            "source": exercise_directory / "solution" / f"{solution}.{ext}",
             "judge": Path(f"{config.rootdir}"),
             "workdir": work_dir,
             "test_suite": suite,
             "options": {"linter": False},
         },
         options,
     )
+
+    # Check if we need to populate the workdir.
+    if (workdir_files := exercise_directory / "workdir").is_dir():
+        shutil.copytree(workdir_files, work_dir, dirs_exist_ok=True)
+
     return get_converter().structure(option_dict, DodonaConfig)
 
 

tests/test_integration_javascript.py

@@ -0,0 +1,95 @@
+"""
+This test will execute the sample solution for every JavaScript exercise to see if
+they pass.
+"""
+
+import json
+import os
+from pathlib import Path
+
+import pytest
+
+from tests.manual_utils import (
+    assert_valid_output,
+    execute_config,
+    exercise_configuration,
+)
+
+LOCAL_REPOSITORY: str | None = os.getenv("EXERCISE_REPO")
+
+# Exercises to exclude, using the exercise folder name.
+BLACKLIST = ["mad libs", "speleologie", "turbolift", "bifidcodering", "blockchain"]
+
+
+def prepare_repository() -> Path:
+    """
+    Download the repository to a temporary folder or use the local repository.
+    :return: Path to the repository.
+    """
+
+    if LOCAL_REPOSITORY:
+        local_path = Path(LOCAL_REPOSITORY)
+        if not local_path.is_dir():
+            raise Exception(
+                f"""
+                Local repository not found. LOCAL_REPOSITORY is set to {LOCAL_REPOSITORY},
+                but this is not a valid directory.
+            """
+            )
+        return local_path
+
+    raise Exception(
+        "Path to local repository not set. Set the EXERCISE_REPO environment variable."
+    )
+
+
+def find_all_exercises(root_directory: Path) -> list[Path]:
+    all_exercises = []
+    for reeks_path in root_directory.glob("*"):
+        if reeks_path.is_dir():
+            for oefening_path in reeks_path.glob("*"):
+                if oefening_path.is_dir():
+                    config_file = oefening_path / "config.json"
+                    if not config_file.exists():
+                        print(f"Skipping {oefening_path}, not an exercise.")
+                        continue
+                    exercise_data = json.loads(config_file.read_text())
+                    if exercise_data.get("evaluation", {}).get("handler") is not None:
+                        print(f"Skipping {oefening_path}, not TESTed.")
+                        continue
+                    if oefening_path.name in BLACKLIST:
+                        print(f"Skipping {oefening_path}, blacklisted.")
+                        continue
+                    all_exercises.append(oefening_path)
+    return all_exercises
+
+
+def get_exercises() -> list[Path]:
+    # Get path to the repo.
+    repository = prepare_repository()
+
+    if not repository:
+        return []
+
+    # Get all exercises in the repository.
+    root_directory = repository / "reeksen"
+    return find_all_exercises(root_directory)
+
+
+ALL_EXERCISES = get_exercises()
+
+
+@pytest.mark.parametrize("exercise", ALL_EXERCISES, ids=lambda ex: ex.name)
+def test_javascript_exercise(exercise: Path, tmp_path: Path, pytestconfig):
+    conf = exercise_configuration(
+        pytestconfig,
+        exercise,
+        "javascript",
+        tmp_path,
+        "suite.yaml",
+        "solution.nl",
+    )
+    result = execute_config(conf)
+    updates = assert_valid_output(result, pytestconfig)
+    status = updates.find_status_enum()
+    assert set(status) == {"correct"}