From 302018b217b8bbae1b7b169bc4b9cd0c36a06f44 Mon Sep 17 00:00:00 2001 From: Josh Mitchell Date: Sat, 22 Jul 2023 01:49:13 +1000 Subject: [PATCH 1/2] Transition from schema to pydantic --- devtools/conda-envs/docs_env.yml | 4 +- devtools/conda-envs/test_env.yml | 2 +- devtools/conda-envs/user_env.yml | 2 +- docs/conf.py | 2 +- docs/configuring.md | 13 +- pyproject.toml | 4 +- soap/__init__.py | 8 +- soap/_cli.py | 28 +-- soap/_models.py | 13 + soap/_soap.py | 10 +- soap/config.py | 410 +++++++++---------------------- 11 files changed, 164 insertions(+), 332 deletions(-) create mode 100644 soap/_models.py diff --git a/devtools/conda-envs/docs_env.yml b/devtools/conda-envs/docs_env.yml index dd3c951..33e6722 100644 --- a/devtools/conda-envs/docs_env.yml +++ b/devtools/conda-envs/docs_env.yml @@ -10,7 +10,7 @@ dependencies: - gitpython - toml - typer[all]>=0.6 - - schema + - pydantic~=2.0.0 - pyyaml # Docs depends @@ -19,4 +19,4 @@ dependencies: - myst-parser~=1.0.0 - sphinx-notfound-page - sphinx-click - - sphinx-jsonschema + - autodoc-pydantic diff --git a/devtools/conda-envs/test_env.yml b/devtools/conda-envs/test_env.yml index d6f9013..298af78 100644 --- a/devtools/conda-envs/test_env.yml +++ b/devtools/conda-envs/test_env.yml @@ -11,7 +11,7 @@ dependencies: - gitpython - toml - typer[all]>=0.6 - - schema + - pydantic~=2.0.0 - pyyaml # Testing diff --git a/devtools/conda-envs/user_env.yml b/devtools/conda-envs/user_env.yml index 3312f81..fac1393 100644 --- a/devtools/conda-envs/user_env.yml +++ b/devtools/conda-envs/user_env.yml @@ -11,5 +11,5 @@ dependencies: - gitpython - toml - typer[all]>=0.6 - - schema + - pydantic~=2.0.0 - pyyaml diff --git a/docs/conf.py b/docs/conf.py index a600fbd..dee4973 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -56,7 +56,7 @@ "sphinx.ext.doctest", "myst_parser", "sphinx_click", - "sphinx-jsonschema", + # "sphinxcontrib.autodoc_pydantic", ] diff --git a/docs/configuring.md b/docs/configuring.md index d0df46d..ddc20e4 100644 --- a/docs/configuring.md +++ b/docs/configuring.md @@ -1,14 +1,3 @@ # Configuring SOAP - -## Environments - -:::{jsonschema} soap.config.ENV_SCHEMA_JSON -::: - - -## Aliases - -:::{jsonschema} soap.config.ALIAS_SCHEMA_JSON -::: - +SOAP is configured with the `soap.toml` file, which uses the fields described in [](soap.config). diff --git a/pyproject.toml b/pyproject.toml index e38d945..d90d435 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ dependencies = [ "gitpython", "toml", "typer[all]>=0.6", - "schema", + "pydantic~=2.0.0", "pyyaml", ] dynamic = ["version"] @@ -37,7 +37,7 @@ docs = [ "myst-parser~=1.0.0", "sphinx-notfound-page", "sphinx-click", - "sphinx-jsonschema", + "autodoc_pydantic", ] all=["soap[test,docs]"] diff --git a/soap/__init__.py b/soap/__init__.py index ab97f6c..a390ed5 100644 --- a/soap/__init__.py +++ b/soap/__init__.py @@ -2,7 +2,7 @@ # Add imports here from soap._soap import prepare_env, run_in_env -from soap.config import Config, Alias, Env +from soap.config import ConfigModel, AliasModel, EnvModel # Import the version from setuptools_scm _version module from soap._version import version as VERSION @@ -11,9 +11,9 @@ __all__ = [ "prepare_env", "run_in_env", - "Config", - "Alias", - "Env", + "ConfigModel", + "AliasModel", + "EnvModel", "VERSION", "VERSION_TUPLE", ] diff --git a/soap/_cli.py b/soap/_cli.py index 0e5abb4..a527068 100644 --- a/soap/_cli.py +++ b/soap/_cli.py @@ -52,11 +52,11 @@ def main(): """Entry point for Snakes on a Plane. Generates commands for aliases and catches and simplifies errors.""" - cfg = soap.Config() - for alias in cfg.aliases: + cfg = soap.ConfigModel.from_file_tree() + for alias_name, alias in cfg.aliases.items(): @app.command( - alias.name, + alias_name, help=alias.description, context_settings={ "allow_extra_args": alias.passthrough_args, @@ -109,14 +109,14 @@ def update( """ Update Conda environments. """ - cfg = soap.Config() - envs = cfg.envs.values() if env is None else [cfg.envs[env]] + cfg = soap.ConfigModel.from_file_tree() + envs = cfg.envs.items() if env is None else [(env, cfg.envs[env])] CONSOLE.print( f"[cyan]Updating {len(envs)} environment{'s' if len(envs) != 1 else ''}" ) - for this_env in envs: + for env_name, this_env in envs: CONSOLE.print( - f"[cyan]Preparing environment '{this_env.name}' " + f"[cyan]Preparing environment '{env_name}' " + f"from '{this_env.yml_path}' " + f"in '{this_env.env_path}'" ) @@ -132,7 +132,7 @@ def run( env: str = Option(DEFAULT_ENV, help="Environment in which to run the command"), ): """Run a command in an environment.""" - cfg = soap.Config() + cfg = soap.ConfigModel.from_file_tree() this_env = cfg.envs[env] soap.prepare_env(this_env) try: @@ -152,7 +152,7 @@ def list( ) ): """List the available environments.""" - cfg = soap.Config() + cfg = soap.ConfigModel.from_file_tree() if verbosity < 3: captions = [ @@ -177,8 +177,8 @@ def list( table.add_column("🪧") table.add_column("📥") - for env in cfg.envs.values(): - row = [env.name, str(env.yml_path)] + for env_name, env in cfg.envs.items(): + row = [env_name, str(env.yml_path)] if verbosity > 0: row.append("✓" if env.install_current else "") if verbosity > 1: @@ -191,8 +191,8 @@ def list( else: tree = rich.tree.Tree("[i]Snakes on a Plane environments", guide_style="dim") - for env in cfg.envs.values(): - branch = tree.add(f"[cyan bold]{env.name}") + for env_name, env in cfg.envs.items(): + branch = tree.add(f"[cyan bold]{env_name}") yml_branch = branch.add(f"[b]YAML path:[/b] {env.yml_path}") branch.add(f"[b]Environment path:[/b] {env.env_path}") environment_exists = ( @@ -211,7 +211,7 @@ def list( yml_branch.add(syntax) if verbosity > 4: branch.add( - f"[b]Command to update environment:[/b] [u]soap update --env {env.name}" + f"[b]Command to update environment:[/b] [u]soap update --env {env_name}" ) branch.add( f"[b]Command to recreate environment:[/b] [u]soap update --recreate --env {env.name}" diff --git a/soap/_models.py b/soap/_models.py new file mode 100644 index 0000000..02230e7 --- /dev/null +++ b/soap/_models.py @@ -0,0 +1,13 @@ +from pydantic import BaseModel as _BaseModel +from pydantic import ConfigDict + +__all__ = [ + "BaseModel", +] + + +class BaseModel(_BaseModel): + model_config = ConfigDict( + validate_assignment=True, + extra="forbid", + ) diff --git a/soap/_soap.py b/soap/_soap.py index 9217d9e..e8bb73c 100644 --- a/soap/_soap.py +++ b/soap/_soap.py @@ -7,7 +7,7 @@ import yaml import soap.conda -from soap.config import Env +from soap.config import EnvModel from soap.utils import yaml_file_to_dict, dict_to_yaml_str @@ -33,7 +33,7 @@ def add_pip_package( dependencies.append({"pip": [package]}) -def prepare_env_file(env: Env) -> str: +def prepare_env_file(env: EnvModel) -> str: """ Prepare an environment YAML file and return its contents @@ -56,7 +56,7 @@ def prepare_env_file(env: Env) -> str: # Add the current package, in dev mode, if required if env.install_current: add_pip_package( - f"-e {env.package_root}[all]", + f"-e {env._package_root}[all]", env_dict["dependencies"], ) @@ -64,7 +64,7 @@ def prepare_env_file(env: Env) -> str: def prepare_env( - env: Env, + env: EnvModel, ignore_cache: bool = False, ): """ @@ -122,7 +122,7 @@ def prepare_env( working_yaml_path.unlink() -def run_in_env(args: Sequence[str], env: Env): +def run_in_env(args: Sequence[str], env: EnvModel): """ Run a command in the provided environment. Does not update the environment. diff --git a/soap/config.py b/soap/config.py index bc30327..3d24a5c 100644 --- a/soap/config.py +++ b/soap/config.py @@ -1,144 +1,138 @@ """Configuration for Snakes on a Plane""" -from typing import Any, Dict, Union, List +from typing import Any, Dict, Union, List, Optional from soap.utils import get_git_root from soap.exceptions import MissingConfigFileError import toml from pathlib import Path -from schema import Schema, Optional, Or, And, Use, Literal +from pydantic import Field, FilePath, field_validator, FieldValidationInfo + +from ._models import BaseModel __all__ = [ - "Config", - "Env", - "Alias", + "ConfigModel", + "EnvModel", + "AliasModel", ] -def _get_name(schema): - while not isinstance(schema, str): - schema = schema.schema - return schema +DEFAULT_ENV = "test" -ROOT_DIR_KEY = "ROOT_DIR" +class EnvModel(BaseModel): + yml_path: FilePath + """Path to YAML file defining the environment.""" + env_path: Optional[Path] = None # TODO: Figure out how to remove the Optional + """Prefix of the new environment. Defaults to `.soap/`""" + install_current: bool = True + """If True, install the current project in this environment after all dependencies.""" + additional_channels: list[str] = Field(default_factory=list) + """Channels to prepend to the environment file's channel list.""" + additional_dependencies: list[str] = Field(default_factory=list) + """Packages and constraints to add to the environment file.""" + + @field_validator("yml_path", "env_path") + def root_paths(cls, value, info: FieldValidationInfo): + if value.is_absolute(): + return value + + if isinstance(info.context, dict): + root_dir = Path(info.context.get("root_dir", ".")) + else: + root_dir = Path() + return root_dir / value -DEFAULT_ENV = "test" +class AliasModel(BaseModel): + command: str = Field(..., alias="cmd") + """The command to run.""" -ENV_SCHEMA = Schema( - { - Literal( - "yml_path", - description="Path to YAML file defining the environment.", - ): And(str, Use(Path)), - Optional( - Literal( - "env_path", - description="Prefix of the new environment. Defaults to `.soap/`", - ), - default=None, - ): And(str, Use(Path)), - Optional( - Literal( - "install_current", - description="If True, install the current project in this environment after all dependencies.", - ), - default=True, - ): bool, - Optional( - Literal( - "additional_channels", - description="Channels to prepend to the environment file's channel list.", - ), - default=[], - ): [str], - Optional( - Literal( - "additional_dependencies", - description="Packages and constraints to add to the environment file.", - ), - default=[], - ): [str], - } -) -ENV_SCHEMA_JSON = ENV_SCHEMA.json_schema("`[envs]`") - -_ENV_DEFAULTS = { - _get_name(key): key.default for key in ENV_SCHEMA.schema if hasattr(key, "default") -} - -ALIAS_SCHEMA = Schema( - { - Literal("cmd", description="The command to alias"): str, - Optional( - Literal( - "chdir", - description=( - "Where to run the command. True for the git repository root" - + " directory, False for the working directory, or a path" - + " relative to the root directory.", - ), - ), - default=False, - ): Or(bool, And(str, Use(Path))), - Optional( - Literal( - "env", - description="The environment in which to run this command when the --env argument is not passed.", - ), - default=DEFAULT_ENV, - ): str, - Optional( - Literal( - "description", - description="A description of this command for the --help argument.", - ), - default="", - ): str, - Optional( - Literal( - "passthrough_args", - description="If True, SOAP will pass any unrecognised arguments through to the aliased command.", - ), - default=False, - ): bool, - } -) -ALIAS_SCHEMA_JSON = ALIAS_SCHEMA.json_schema("`[aliases]`") - -_ALIAS_DEFAULTS = { - _get_name(key): key.default - for key in ALIAS_SCHEMA.schema - if hasattr(key, "default") -} - -CFG_SCHEMA = Schema( - { - Optional("envs", default={}): Or( - {}, - { - str: Or( - And(str, Use(lambda s: _ENV_DEFAULTS | {"yml_path": Path(s)})), - ENV_SCHEMA, - ) - }, - ), - Optional("aliases", default={}): Or( - {}, - { - str: Or( - And(str, Use(lambda s: _ALIAS_DEFAULTS | {"cmd": s})), - ALIAS_SCHEMA, - ) - }, - ), - } -) -CFG_SCHEMA_JSON = CFG_SCHEMA.json_schema("Snakes On A Plane: `soap.toml`") - - -def _get_cfg_map(leaf_path: Union[None, Path] = None) -> Dict[str, Any]: + chdir: Union[bool, Path] = False + """ + Where to run the command. + + True for the git repository root directory, False for the working directory, + or a path relative to the root directory. + """ + + default_env: str = Field(DEFAULT_ENV, alias="env") + """The environment in which to run this command when the --env argument is not passed.""" + + description: str = "" + """A description of this command for the --help argument.""" + + passthrough_args: bool = False + """If True, SOAP will pass any unrecognised arguments through to the aliased command.""" + + @field_validator("chdir") + def set_chdir(cls, value, info: FieldValidationInfo): + if isinstance(info.context, dict): + root_dir = Path(info.context.get("root_dir", ".")) + else: + root_dir = Path() + + if not value: + return value + elif value is True: + return root_dir + else: + return root_dir / value + + +class ConfigModel(BaseModel): + envs: dict[str, EnvModel] = Field(default_factory=dict) + """Environments to run commands in.""" + aliases: dict[str, AliasModel] = Field(default_factory=dict) + """Aliases for commands.""" + + @classmethod + def from_file_tree( + cls, + leaf_path: Union[Path, None] = None, + cfg: Union[Dict[str, Any], None] = None, + ): + if cfg is not None: + return cls.model_validate(cfg) + + root_dir, cfg = _get_cfg_map(leaf_path) + + return cls.model_validate(cfg, context={"root_dir": root_dir}) + + @field_validator("envs", mode="before") + def proc_string_envs(cls, value, info: FieldValidationInfo): + try: + return { + name: {"yml_path": value} if isinstance(value, str) else value + for name, value in value.items() + } + except AttributeError: + return value + + @field_validator("envs", mode="after") + def set_env_paths(cls, value, info: FieldValidationInfo): + if isinstance(info.context, dict): + root_dir = Path(info.context.get("root_dir", ".")) + else: + root_dir = Path() + for name, model in value.items(): + if model.env_path is None: + model.env_path = root_dir / ".soap" / name + model._package_root = root_dir + return value + + @field_validator("aliases", mode="before") + def proc_string_aliases(cls, value, info: FieldValidationInfo): + try: + return { + name: {"cmd": value} if isinstance(value, str) else value + for name, value in value.items() + } + except AttributeError: + return value + + +def _get_cfg_map(leaf_path: Union[None, Path] = None) -> tuple[Path, Dict[str, Any]]: """ Get the configuration map for SOAP @@ -196,7 +190,7 @@ def _get_cfg_map(leaf_path: Union[None, Path] = None) -> Dict[str, Any]: ) # Raise an error if we haven't found any config files - if pyproject_path is None and len(soaptoml_paths) == 0: + if root_path is None: raise MissingConfigFileError( f"Couldn't find pyproject.toml or any soap.toml files searching up" + f" the file tree from {leaf_path}" @@ -209,15 +203,7 @@ def _get_cfg_map(leaf_path: Union[None, Path] = None) -> Dict[str, Any]: pyproject_data.get("tool", {}).get("soap", {}) data = _combine_cfg_maps([data, pyproject_data]) - # Validate against the schema - data = CFG_SCHEMA.validate(data) - - # Make sure we haven't added an entry with this name to the config schema, - # then add the root path to the data - assert ROOT_DIR_KEY not in (k.key for k in CFG_SCHEMA.schema) - data[ROOT_DIR_KEY] = root_path - - return data + return root_path, data def _combine_cfg_maps(data: List[Dict[str, None]]): @@ -226,159 +212,3 @@ def _combine_cfg_maps(data: List[Dict[str, None]]): At the moment this just returns the first dictionary.""" # TODO: Combine multiple dictionaries return data[0] - - -class Config: - """ - Configuration for a SOAP project - - Attributes - ========== - - envs - Mapping from environment names to environment ``Env`` objects. The map - key matches the ``.name`` attribute of the environment. - aliases - List of ``Alias`` objects defined in the project. - - Raises - ====== - - MissingConfigFileError - If ``pyproject.toml`` and ``soap.toml`` are both missing from the root - path. - TomlDecodeError - If the file was found but is not valid TOML - SchemaError - If the config file is valid TOML but does not match the SOAP configuration - schema. - """ - - def __init__( - self, - leaf_path: Union[Path, None] = None, - cfg: Union[Dict[str, Any], None] = None, - ): - if cfg is None: - cfg = _get_cfg_map(leaf_path) - - self.root_dir = cfg[ROOT_DIR_KEY] - self.envs = { - name: Env(name, value, self.root_dir) for name, value in cfg["envs"].items() - } - self.aliases = [ - Alias(name, value, self.root_dir) for name, value in cfg["aliases"].items() - ] - - def __repr__(self): - return f"" - - -class Env: - """ - Configuration for a single environment. - - Attributes - ========== - - name - Name of the environment - yml_path - Path to the Conda environment YAML file - env_path - Path to the environment prefix - install_current - True if the current project should be installed - in the environment with ``pip install -e .`` - """ - - def __init__( - self, - name: str, - value: Dict[str, Any], - package_root: Path, - ): - self.name: str = name - self.package_root = Path(package_root).absolute() - - self.yml_path = ( - value["yml_path"] - if value["yml_path"].is_absolute() - else self.package_root / value["yml_path"] - ) - - self.env_path = ( - Path(".soap") / self.name - if value["env_path"] is None - else value["env_path"] - ) - if not self.env_path.is_absolute(): - self.env_path = self.package_root / self.env_path - - self.install_current = value["install_current"] - self.additional_channels = value["additional_channels"] - self.additional_dependencies = value["additional_dependencies"] - - def __repr__(self): - return ( - f"Env(name={self.name!r}, value={{" - + f"yml_path: {self.yml_path!r}, " - + f"env_path: {self.env_path!r}, " - + f"package_root: {self.package_root!r}, " - + f"install_current: {self.install_current!r}," - + f"additional_channels: {self.additional_channels!r}," - + f"additional_dependencies: {self.additional_dependencies!r}," - + f"}})" - ) - - -class Alias: - """ - Configuration for a single alias. - - Attributes - ========== - - name - Name of the alias. This is the subcommand used to execute the alias. - command - The command being aliased. - chdir - If True, the command will be run from the Git repository root directory, - rather than the current directory. - default_env - The environment to run the alias in if none is specified on the command - line. - """ - - def __init__( - self, - name, - value: Dict[str, Any], - root_dir: Path, - ): - self.name = name - self.command = value["cmd"] - - self.chdir: Path | None - if not value["chdir"]: - self.chdir = None - elif value["chdir"] is True: - self.chdir = root_dir - else: - self.chdir = root_dir / value["chdir"] - - self.default_env = value["env"] - self.description = value["description"] or "Alias for `" + self.command + "`" - self.passthrough_args = value["passthrough_args"] - - def __repr__(self): - return ( - f"Alias(name={self.name!r}, value={{" - + f"cmd: {self.command!r}, " - + f"chdir: {self.chdir!r}, " - + f"env: {self.default_env!r}, " - + f"description: {self.description!r}}}, " - + f"passthrough_args: {self.passthrough_args!r}" - + f"}})" - ) From 6576c12b87dfc37b0a7cdf02d01d2feb790900f2 Mon Sep 17 00:00:00 2001 From: Josh Mitchell Date: Sat, 22 Jul 2023 01:59:16 +1000 Subject: [PATCH 2/2] Progress on tests --- soap/tests/test_config.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/soap/tests/test_config.py b/soap/tests/test_config.py index caaded2..f90d848 100644 --- a/soap/tests/test_config.py +++ b/soap/tests/test_config.py @@ -78,20 +78,18 @@ def test_maximalist_toml(): """Check that a maximalist toml file validates correctly.""" data = toml.load("soap/tests/data/maximalist.toml") - validated = cfg.CFG_SCHEMA.validate(data) - assert validated == MAXIMALIST_VALIDATED + validated = cfg.ConfigModel.model_validate(data) + assert validated.model_dump() == MAXIMALIST_VALIDATED def test_maximalist_cfg(): root_dir = Path("/home/someone/project") config_dict: Any = deepcopy(MAXIMALIST_VALIDATED) - config_dict[cfg.ROOT_DIR_KEY] = root_dir - config = cfg.Config(cfg=config_dict) + config = cfg.ConfigModel.model_validate(config_dict, context={"root_dir": root_dir}) assert config.envs["test"].env_path == root_dir / ".soap/test" assert config.envs["docs"].env_path == root_dir / ".soap/docs" assert config.envs["user"].env_path == Path("/home/someone/conda/envs/soap-env") - alias_dict = {alias.name: alias for alias in config.aliases} - assert alias_dict["docs"].chdir == root_dir + assert config.aliases["docs"].chdir == root_dir