Add better CLI

README.md

@@ -7,6 +7,11 @@ If you use this software in research, please cite:
 
 - Strijbol, N., Van Petegem, C., Maertens, R., Sels, B., Scholliers, C., Dawyndt, P., & Mesuere, B. (2023). TESTed—An educational testing framework with language-agnostic test suites for programming exercises. SoftwareX, 22, 101404. [doi:10.1016/j.softx.2023.101404](https://doi.org/10.1016/j.softx.2023.101404)
 
+> [!IMPORTANT]
+> The documentation below is intended for running TESTed as a standalone tool.
+> If you are looking to create exercises for Dodona, we have [more suitable documentation available](https://docs.dodona.be/nl/guides/exercises/).
+
+
 ## Installing TESTed
 
 TESTed is implemented in Python, but has various dependencies for its language-specific modules.
@@ -162,23 +167,56 @@ optional arguments:
 Adjust the configuration file if you want to evaluate the wrong submission.
 
 For reference, the file `tested/dsl/schema.json` contains the JSON Schema of the test suite format.
-Some other useful commands are:
+
+## Running TESTed locally
+
+The `python -m tested` command is intended for production use.
+However, it is not always convenient to create a `config.json` file for each exercise to run.
+
+TESTed supports two ways of running TESTed without a config file.
+The first way is:
 
 ```bash
 # Run a hard-coded exercise with logs enabled, useful for debugging
 $ python -m tested.manual
-# Convert a YAML test suite into JSON
-$ python -m tested.dsl
 ```
 
+This command is useful when debugging TESTed itself or a particularly challenging exercise.
+It will execute a hardcoded config, which is set in `tested/manual.py`.
+
+The second way is:
+
+```bash
+# Run an exercise with CLI paramaters
+$ python -m tested.cli --help
+usage: cli.py [-h] -e EXERCISE [-s SUBMISSION] [-t TESTSUITE] [-f] [-p PROGRAMMING_LANGUAGE]
+
+Simple CLI for TESTed
+
+options:
+  -h, --help            show this help message and exit
+  -e EXERCISE, --exercise EXERCISE
+                        Path to a directory containing an exercise
+  -s SUBMISSION, --submission SUBMISSION
+                        Path to a submission to evaluate
+  -t TESTSUITE, --testsuite TESTSUITE
+                        Path to a test suite
+  -f, --full            If the output should be shown in full (default: false)
+  -p PROGRAMMING_LANGUAGE, --programming_language PROGRAMMING_LANGUAGE
+                        The programming language to use
+
+additional information: The CLI only looks at a config.json file in the exercise directory. It does not look in folders above the exercise directory.
+```
+
+This is the "CLI mode": here you can pass various options as command line parameters.
+For example, for exercises following a standardized directory structure, the path to the exercise folder is often enough.
+
 ## TESTed repository
 
 The repository of TESTed is organized as follows:
 
-- `exercise`: exercises with preconfigured test suites; useful to play around with TESTed and also used by unit tests for TESTed itself
 - `tested`: Python code of the actual judge (run by Dodona)
 - `tests`: unit tests for TESTed
-- `benchmarking`: utilities to benchmark TESTed
 
 You can run the basic unit tests with:
 

tested/cli.py

@@ -0,0 +1,240 @@
+"""
+A module that provides a simpler CLI to TESTed.
+"""
+import json
+import os
+import re
+import shutil
+import textwrap
+import time
+from argparse import ArgumentParser, ArgumentTypeError
+from io import StringIO
+from pathlib import Path
+
+from tested.configs import DodonaConfig, GlobalConfig
+from tested.languages import get_language
+from tested.main import run
+from tested.testsuite import Suite, SupportedLanguage
+
+
+def dir_path(path):
+    if os.path.isdir(path):
+        return Path(path)
+    else:
+        raise ArgumentTypeError(f"{path} is not a valid path (cwd: {os.getcwd()})")
+
+
+def find_submission() -> Path:
+    # First, if we have an explicit submission, use that.
+    if args.submission is not None:
+        return Path(args.submission)
+    # Attempt to determine the programming language of this exercise.
+    considered = []
+    # There is a bit of a chicken-and-egg problem here.
+    # We need the language to complete the config, but the language needs a config.
+    # We thus create a dummy config.
+    global_config = GlobalConfig(
+        dodona=DodonaConfig(
+            resources=Path("."),
+            source=Path("."),
+            time_limit=10,
+            memory_limit=10,
+            natural_language="none",
+            programming_language=SupportedLanguage(programming_language),
+            workdir=Path("."),
+            judge=Path("."),
+        ),
+        testcase_separator_secret="",
+        context_separator_secret="",
+        suite=Suite(),
+    )
+    language = get_language(global_config, programming_language)
+    solution_folder = exercise_path / "solution"
+    possible_submissions = list(solution_folder.glob(f"*.{language.file_extension()}"))
+
+    if len(possible_submissions) == 0:
+        raise FileNotFoundError(
+            f"Could not find a submission file in {solution_folder}/*.{language.file_extension()}.\n"
+            "Please ensure a submission file exists in the right location or provide an alternative path via the --submission parameter on the command line."
+        )
+
+    return possible_submissions[0]
+
+
+def create_and_populate_workdir(config: DodonaConfig):
+    # Create workdir if needed.
+    config.workdir.mkdir(exist_ok=True)
+
+    # Delete content in work dir
+    for root, dirs, files in os.walk(config.workdir):
+        for f in files:
+            os.unlink(os.path.join(root, f))
+        for d in dirs:
+            shutil.rmtree(os.path.join(root, d), ignore_errors=True)
+
+    exercise_workdir = config.resources.parent / "workdir"
+
+    # Copy existing files to workdir if needed.
+    if exercise_workdir.is_dir():
+        shutil.copytree(exercise_workdir, config.workdir, dirs_exist_ok=True)
+
+
+def split_output(to_split: str) -> list[str]:
+    return re.compile(r"(?<=})\s*(?={)").split(to_split)
+
+
+class CommandDict(list):
+    def find_all(self, command: str) -> list[dict]:
+        return [x for x in self if x["command"] == command]
+
+    def find_next(self, command: str) -> dict:
+        return next(x for x in self if x["command"] == command)
+
+    def find_status_enum(self) -> list[str]:
+        commands = [
+            x
+            for x in self
+            if x["command"].startswith("close-") or x["command"] == "escalate-status"
+        ]
+        return [x["status"]["enum"] for x in commands if "status" in x]
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser(
+        description="Simple CLI for TESTed",
+        epilog=textwrap.dedent(
+            """
+                 additional information:\n
+                     The CLI only looks at a config.json file in the exercise directory.\n
+                     It does not look in folders above the exercise directory.
+                 """
+        ),
+    )
+    parser.add_argument(
+        "-e",
+        "--exercise",
+        type=dir_path,
+        help="Path to a directory containing an exercise",
+        required=True,
+    )
+    parser.add_argument(
+        "-s",
+        "--submission",
+        type=Path,
+        help="Path to a submission to evaluate",
+        default=None,
+    )
+    parser.add_argument(
+        "-t",
+        "--testsuite",
+        type=Path,
+        help="Path to a test suite",
+        default=None,
+    )
+    parser.add_argument(
+        "-f",
+        "--full",
+        action="store_false",
+        help="If the output should be shown in full (default: false)",
+        default=None,
+    )
+    parser.add_argument(
+        "-p",
+        "--programming_language",
+        help="The programming language to use",
+        default=None,
+    )
+    args = parser.parse_args()
+
+    exercise_path = args.exercise
+    evaluation_path = exercise_path / "evaluation"
+    judge_path = Path(__file__).parent.parent
+    config_path = exercise_path / "config.json"
+
+    try:
+        config_file = json.loads(config_path.read_text())
+    except FileNotFoundError:
+        config_file = dict()
+
+    if args.programming_language is not None:
+        programming_language = args.programming_language
+    else:
+        try:
+            programming_language = config_file["programming_language"]
+        except KeyError:
+            if config_path.exists():
+                raise Exception(
+                    f"Could not determine the programming language for {exercise_path}, as there was no programming language in the config file (at {config_path}).\n"
+                    "Please add a programming language to the config file or provide the programming language via the --programming_language parameter on the command line."
+                )
+            else:
+                raise Exception(
+                    f"Could not determine the programming language for {exercise_path}, as there was no config file (at {config_path}) for this exercise.\n"
+                    "Please create a config file or provide the programming language via the --programming_language parameter on the command line."
+                )
+
+    if args.testsuite is not None:
+        suite_path = args.testsuite.relative_to(evaluation_path)
+    elif "evaluation" in config_file and "test_suite" in config_file["evaluation"]:
+        # The config file contains a location, so try to use it.
+        suite_path = evaluation_path / config_file["evaluation"]["test_suite"]
+        if not suite_path.is_file():
+            raise FileNotFoundError(
+                f"The test suite at {suite_path} does not exist (read value from the config file).\n"
+                "Create the file, correct the config.json file or provide the path to the test suite via the --testsuite parameter on the command line."
+            )
+    else:
+        suite_path = evaluation_path / "suite.yaml"
+        if not suite_path.is_file():
+            raise FileNotFoundError(
+                f"The test suite at {suite_path} does not exist (used default value).\n"
+                "Create the file, add the location to the config.json file or provide the path to the test suite via the --testsuite parameter on the command line."
+            )
+    submission_path = find_submission()
+    workdir_path = judge_path / "workdir"
+
+    dodona_config = DodonaConfig(
+        resources=evaluation_path,
+        source=submission_path,
+        time_limit=60,
+        memory_limit=536870912,
+        natural_language="nl",
+        programming_language=SupportedLanguage(programming_language),
+        workdir=judge_path / "workdir",
+        judge=judge_path,
+        test_suite=suite_path,
+    )
+
+    output_handler = StringIO()
+
+    create_and_populate_workdir(dodona_config)
+
+    print(f"Locally executing exercise {exercise_path}...")
+    print("The following options will be used:")
+    print(f" - Test suite: {suite_path}")
+    print(f" - Submission: {submission_path}")
+    print("")
+    print(
+        f"The execution will happen in {workdir_path}. This folder will remain available after execution for inspection."
+    )
+
+    start = time.time()
+    run(dodona_config, output_handler)
+    end = time.time()
+    print()
+    print(f"Execution took {end - start} seconds (real time).")
+
+    if args.full:
+        print("Results:")
+        print(output_handler.getvalue())
+    else:
+        output = output_handler.getvalue()
+        updates = CommandDict()
+        # Every update should be valid.
+        for update in split_output(output):
+            update_object = json.loads(update)
+            updates.append(update_object)
+
+        results = updates.find_status_enum()
+        correct_enums = len([x for x in results if x == "correct"])
+        print(f"{correct_enums} of {len(results)} testcases were correct.")

tested/manual.py

@@ -3,17 +3,16 @@
 file, allowing rapid testing (and, most importantly, debugging).
 """
 import logging
-import os
-import shutil
 import sys
 import time
 from pathlib import Path
 
+from tested.cli import create_and_populate_workdir
 from tested.configs import DodonaConfig, Options
 from tested.main import run
 from tested.testsuite import SupportedLanguage
 
-exercise_dir = "/home/niko/Ontwikkeling/universal-judge/tests/exercises/global"
+exercise_dir = "/home/niko/Ontwikkeling/universal-judge/tests/exercises/echo"
 
 
 def read_config() -> DodonaConfig:
@@ -51,21 +50,7 @@
     logger = logging.getLogger("tested.parsing")
     logger.setLevel(logging.INFO)
 
-    # Create workdir if needed.
-    config.workdir.mkdir(exist_ok=True)
-
-    # Delete content in work dir
-    for root, dirs, files in os.walk(config.workdir):
-        for f in files:
-            os.unlink(os.path.join(root, f))
-        for d in dirs:
-            shutil.rmtree(os.path.join(root, d), ignore_errors=True)
-
-        # Copy existing files to workdir if needed.
-    if Path(exercise_dir, "workdir").is_dir():
-        shutil.copytree(
-            Path(exercise_dir, "workdir"), config.workdir, dirs_exist_ok=True
-        )
+    create_and_populate_workdir(config)
 
     start = time.time()
     run(config, sys.stdout)

tests/manual_utils.py

@@ -1,37 +1,17 @@
 import json
-import re
 from io import StringIO
 from pathlib import Path
 
 from jsonschema import validate
 
+from tested.cli import CommandDict, split_output
 from tested.configs import DodonaConfig
 from tested.languages import get_language
 from tested.main import run
 from tested.parsing import get_converter
 from tested.utils import recursive_dict_merge
 
 
-def split_output(output: str) -> list[str]:
-    return re.compile(r"(?<=})\s*(?={)").split(output)
-
-
-class CommandDict(list):
-    def find_all(self, command: str) -> list[dict]:
-        return [x for x in self if x["command"] == command]
-
-    def find_next(self, command: str) -> dict:
-        return next(x for x in self if x["command"] == command)
-
-    def find_status_enum(self) -> list[str]:
-        commands = [
-            x
-            for x in self
-            if x["command"].startswith("close-") or x["command"] == "escalate-status"
-        ]
-        return [x["status"]["enum"] for x in commands if "status" in x]
-
-
 def assert_valid_output(output: str, config) -> CommandDict:
     with open(Path(config.rootdir) / "tests/partial_output.json", "r") as f:
         schema = json.load(f)