Merge pull request #411 from dodona-edu/feature/oracles

Rename evaluator to oracle

.editorconfig

@@ -13,6 +13,9 @@ ij_smart_tabs = false
 ij_visual_guides = 84
 ij_wrap_on_typing = false
 
+[*.nix]
+indent_size = 2
+
 [{*.json,*.yaml,*.yml}]
 indent_size = 2
 ij_json_keep_blank_lines_in_code = 0

.github/workflows/ci.yml

@@ -40,7 +40,7 @@ jobs:
       - run: sudo apt -y install hlint cppcheck shellcheck checkstyle
       - run: curl -sSLO https://github.com/pinterest/ktlint/releases/download/0.48.2/ktlint && chmod a+x ktlint
       - run: echo "${GITHUB_WORKSPACE}" >> $GITHUB_PATH
-      - run: pipenv run pytest --cov=tested tests/
+      - run: pipenv run pytest -n auto --cov=tested tests/
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
   lint:

Pipfile

@@ -4,12 +4,13 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-psutil = "==5.9.4"
-pydantic = "==1.9.2"
-jsonschema = "==4.17.3"
-typing-inspect = "==0.8.0"
+psutil = "==5.9.5"
+attrs = "==22.2.0"
+cattrs = "==23.1.2"
+jsonschema = "==4.18.4"
+typing-inspect = "==0.9.0"
 pyyaml = "==6.0"
-pygments = "==2.14.0"
+pygments = "==2.15.1"
 python-i18n = "==0.3.9"
 # Required for the Python language module
 pylint = "==2.17.1"
@@ -22,6 +23,7 @@ marko = "==2.0.0"
 pytest = ""
 pytest-mock = ""
 pytest-cov = ""
+pytest-xdist = ""
 
 [requires]
 python_version = "3.11"

README.md

@@ -165,8 +165,6 @@ For reference, the file `tested/dsl/schema.json` contains the JSON Schema of the
 Some other useful commands are:
 
 ```bash
-# Prints the JSON Schema of the (JSON) test suite format
-$ python -m tested.testsuite
 # Run a hard-coded exercise with logs enabled, useful for debugging
 $ python -m tested.manual
 # Convert a YAML test suite into JSON

flake.nix

@@ -36,53 +36,33 @@
             maintainers = [];
           };
         };
-        my-pydantic = python.pkgs.buildPythonPackage rec {
-             pname = "pydantic";
-             version = "1.9.2";
-             format = "setuptools";
-
-             src = pkgs.fetchPypi {
-               inherit pname version;
-               hash = "sha256-jLC8UJv7cTBdelnQAWPV+fxFMPCIHqMsdP9PdMhfPT0=";
-             };
-
-             propagatedBuildInputs = [
-                python.pkgs.typing-extensions
-             ];
-
-             doCheck = false;
-
-             meta = with pkgs.lib; {
-               homepage = "https://github.com/pydantic/pydantic/";
-               license = licenses.mit;
-               maintainers = [ ];
-             };
-           };
         marko = python.pkgs.buildPythonPackage rec {
-            pname = "marko";
-            version = "2.0.0";
-            format = "pyproject";
-            
-            src = pkgs.fetchPypi {
-                inherit pname version;
-                hash = "sha256-78JkYIkyUME3UQJa6SAuuxOJiHA2/A35AJxquHVGcDA=";
-            };
+          pname = "marko";
+          version = "2.0.0";
+          format = "pyproject";
+
+          src = pkgs.fetchPypi {
+            inherit pname version;
+            hash = "sha256-78JkYIkyUME3UQJa6SAuuxOJiHA2/A35AJxquHVGcDA=";
+          };
 
-            nativeBuildInputs = [
-                python.pkgs.pdm-pep517 python.pkgs.pdm-backend
-            ];
+          nativeBuildInputs = [
+            python.pkgs.pdm-pep517 python.pkgs.pdm-backend
+          ];
 
-            doCheck = false;
+          doCheck = false;
 
-            meta = with pkgs.lib; {
-                homepage = "https://github.com/frostming/marko";
-                license = licenses.mit;
-                maintainers = [ ];
-            };
+          meta = with pkgs.lib; {
+            homepage = "https://github.com/frostming/marko";
+            license = licenses.mit;
+            maintainers = [ ];
+          };
         };
         core-packages = ps: with ps; [
             psutil
-            my-pydantic
+            pydantic
+            attrs
+            cattrs
             jsonschema
             typing-inspect
             pyyaml
@@ -94,6 +74,7 @@
             ps.pytest
             ps.pytest-mock
             ps.pytest-cov
+            ps.pytest-xdist
             # For Pycharm
             ps.setuptools
             ps.isort

tested/configs.py

@@ -1,16 +1,13 @@
 """
 Module for handling and bundling various configuration options for TESTed.
 """
-import dataclasses
-import json
 import logging
-from dataclasses import field
 from pathlib import Path
 from typing import IO, TYPE_CHECKING, Any, Dict, Optional, Tuple
 
-from pydantic import BaseModel, root_validator
-from pydantic.dataclasses import dataclass
+from attrs import define, evolve, field
 
+from tested.parsing import fallback_field, get_converter
 from tested.testsuite import ExecutionMode, Suite
 from tested.utils import consume_shebang, get_identifier, smart_close
 
@@ -21,7 +18,7 @@
 _logger = logging.getLogger(__name__)
 
 
-@dataclass(frozen=True)
+@define(frozen=True)
 class Options:
     """
     TESTed-specific options. Putting these options in the exercise config allows
@@ -44,7 +41,7 @@ class Options:
     fails. If nothing is given, the language-dependent default is used. If a boolean
     is given, this value is used, regardless of the language default.
     """
-    language: Dict[str, Dict[str, Any]] = field(default_factory=dict)
+    language: Dict[str, Dict[str, Any]] = field(factory=dict)
     """
     Language-specific options for the judge. These depend on the language
     implementation; the judge itself does nothing with it.
@@ -56,7 +53,7 @@ class Options:
     """
     optimized: bool = True
     """
-    If the custom Python evaluator should be optimized or not.
+    If the Python oracles should be optimized or not.
     """
     compiler_optimizations: bool = False
     """
@@ -69,7 +66,9 @@ class Options:
     """
 
 
-class DodonaConfig(BaseModel):
+@fallback_field(get_converter(), {"testplan": "test_suite", "plan_name": "test_suite"})
+@define
+class DodonaConfig:
     resources: Path
     source: Path
     time_limit: int
@@ -98,16 +97,6 @@ def linter(self) -> bool:
             return self.options.linter
         return local_config
 
-    # noinspection PyMethodParameters
-    @root_validator(pre=True)
-    def backward_compatibility_for_test_suite(cls, values):
-        compatible_names = ["testplan", "plan_name"]
-        for old_name in compatible_names:
-            if old_name in values:
-                values["test_suite"] = values[old_name]
-                del values[old_name]
-        return values
-
 
 def read_config(config_in: IO) -> DodonaConfig:
     """
@@ -116,17 +105,12 @@ def read_config(config_in: IO) -> DodonaConfig:
     """
     with smart_close(config_in) as input_:
         config_json = input_.read()
-    config_ = json.loads(config_json)
-
-    # Replace the judge directory.
-    parsed: DodonaConfig = DodonaConfig.parse_obj(config_)
-    # noinspection PyDataclass
-    parsed = parsed.copy(update={"judge": parsed.judge})
 
+    parsed = get_converter().loads(config_json, DodonaConfig)
     return parsed
 
 
-@dataclasses.dataclass
+@define
 class GlobalConfig:
     dodona: DodonaConfig
     testcase_separator_secret: str
@@ -138,7 +122,7 @@ def options(self) -> Options:
         return self.dodona.options
 
 
-@dataclasses.dataclass
+@define
 class Bundle:
     """A bundle of arguments and configs for running everything."""
 
@@ -203,8 +187,7 @@ def create_bundle(
     if language is None:
         language, offset = _get_language(config)
         config.source_offset = offset
-    # noinspection PyDataclass
-    adjusted_config = config.copy(update={"programming_language": language})
+    adjusted_config = evolve(config, programming_language=language)
     global_config = GlobalConfig(
         dodona=adjusted_config,
         testcase_separator_secret=get_identifier(),