diff --git a/docs/api/stream_daq.md b/docs/api/stream_daq.md index 4ba5dfd0..d10eec58 100644 --- a/docs/api/stream_daq.md +++ b/docs/api/stream_daq.md @@ -2,7 +2,7 @@ This module is a data acquisition module that captures video streams from Miniscopes based on the `Miniscope-SAMD-Framework` firmware. The firmware repository will be published in future updates but is currently under development and private. ## Command -After [installation](../guide/installation.md) and customizing [device configurations](stream-dev-config) and [runtime configuration](models/config.md) if necessary, run the command described in [CLI Usage](../cli/main.rst). +After [installation](../guide/installation.md) and customizing [device configurations](stream-dev-config) and [runtime configuration](models/config.md) if necessary, run the command described in [CLI Usage](../cli/index). One example of this command is the following: ```bash diff --git a/docs/cli/config.md b/docs/cli/config.md new file mode 100644 index 00000000..b75d8791 --- /dev/null +++ b/docs/cli/config.md @@ -0,0 +1,5 @@ +# `config` + +```{click} miniscope_io.cli.config:config +:prog: mio config +``` \ No newline at end of file diff --git a/docs/cli/main.rst b/docs/cli/index.md similarity index 79% rename from docs/cli/main.rst rename to docs/cli/index.md index 2af4c3e3..e5863aa8 100644 --- a/docs/cli/main.rst +++ b/docs/cli/index.md @@ -1,5 +1,10 @@ -CLI Usage -========= +# CLI Usage + +```{toctree} +config +stream +update +``` Refer to the following page for details regarding ``stream_daq`` device config files. diff --git a/docs/cli/stream.md b/docs/cli/stream.md new file mode 100644 index 00000000..1fe621cd --- /dev/null +++ b/docs/cli/stream.md @@ -0,0 +1,5 @@ +# `stream` + +```{click} miniscope_io.cli.stream:stream +:prog: mio stream +``` \ No newline at end of file diff --git a/docs/cli/update.md b/docs/cli/update.md new file mode 100644 index 00000000..f372fa13 --- /dev/null +++ b/docs/cli/update.md @@ -0,0 +1,5 @@ +# `update` + +```{click} miniscope_io.cli.update:update +:prog: mio update +``` \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 17452239..bd9e69f9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,9 +28,11 @@ "sphinx.ext.napoleon", "sphinx.ext.autodoc", "sphinxcontrib.autodoc_pydantic", + "sphinxcontrib.programoutput", "sphinx.ext.intersphinx", "sphinx.ext.todo", "sphinx_click", + "sphinx_design", ] templates_path = ["_templates"] diff --git a/docs/guide/config.md b/docs/guide/config.md new file mode 100644 index 00000000..10fc3299 --- /dev/null +++ b/docs/guide/config.md @@ -0,0 +1,157 @@ +# Configuration + +```{tip} +See also the API docs in {mod}`miniscope_io.models.config` +``` + +Config in `miniscope-io` uses a combination of pydantic models and +[`pydantic-settings`](https://docs.pydantic.dev/latest/concepts/pydantic_settings/). + +Configuration takes a few forms: + +- **Global config:** control over basic operation of `miniscope-io` like logging, + location of user directories, plugins, etc. +- **Device config:** control over the operation of specific devices and miniscopes like + firmware versions, ports, capture parameters, etc. +- **Runtime/experiment config:** control over how a device behaves when it runs, like + plotting, data output, etc. + +## Global Config + +Global config uses the {class}`~miniscope_io.models.config.Config` class + +Config values can be set (in order of priority from high to low, where higher +priorities override lower priorities) + +* in the arguments passed to the class constructor (not user configurable) +* in environment variables like `export MINISCOPE_IO_LOG_DIR=~/` +* in a `.env` file in the working directory +* in a `mio_config.yaml` file in the working directory +* in the `tool.miniscope_io.config` table in a `pyproject.toml` file in the working directory +* in a user `mio_config.yaml` file in the user directory (see [below](user-directory)) +* in the global `mio_config.yaml` file in the platform-specific data directory + (use `mio config global path` to find its location) +* the default values in the {class}`~miniscope_io.models.config.Config` model + +Parent directories are _not_ checked - `.env` files, `mio_config.yaml`, and `pyproject.toml` +files need to be in the current working directory to be discovered. + +You can see your current configuration with `mio config` + +(user-directory)= +### User Directory + +The configuration system allows project-specific configs per-directory with +`mio_config.yaml` files in the working directory, as well as global configuration +via `mio_config.yaml` in the system-specific config directory +(via [platformdirs](https://pypi.org/project/platformdirs/)). +By default, `miniscope-io` does not create new directories in the user's home directory +to be polite, but the site config directory might be inconvenient or hard to reach, +so it's possible to create a user directory in a custom location. + +`miniscope_io` discovers this directory from the `user_dir` setting from +any of the available sources, though the global `mio_config.yaml` file is the most reliable. + +To create a user directory, use the `mio config user create` command. +(ignore the `--dry-run` flag, which are just used to avoid +overwriting configs while rendering the docs ;) + +```{command-output} mio config user create ~/my_new_directory --dry-run +``` + +You can confirm that this will be where miniscope_io discovers the user directory like + +```{command-output} mio config user path +``` + +If a directory is not supplied, the default `~/.config/miniscope_io` is used: + +```{command-output} mio config user create --dry-run +``` + +### Setting Values + +```{todo} +Implement setting values from CLI. + +For now, please edit the configuration files directly. +``` + +### Keys + +#### Prefix + +Keys for environment variables (i.e. set in a shell with e.g. `export` or in a `.env` file) +are prefixed with `MINISCOPE_IO_` to not shadow other environment variables. +Keys in `toml` or `yaml` files are not prefixed with `MINISCOPE_IO_` . + +#### Nesting + +Keys for nested models are separated by a `__` double underscore in `.env` +files or environment variables (eg. `MINISCOPE_IO_LOGS__LEVEL`) + +Keys in `toml` or `yaml` files do not have a dunder separator because +they can represent the nesting directly (see examples below) + +When setting values from the cli, keys for nested models are separated with a `.`. + +#### Case + +Keys are case-insensitive, i.e. these are equivalent:: + + export MINISCOPE_IO_LOGS__LEVEL=INFO + export miniscope_io_logs__level=INFO + +### Examples + +`````{tab-set} +````{tab-item} mio_config.yaml +```{code-block} yaml +user_dir: ~/.config/miniscope_io +log_dir: ~/.config/miniscope_io/logs +logs: + level_file: INFO + level_stream: WARNING + file_n: 5 +``` +```` +````{tab-item} env vars +```{code-block} bash +export MINISCOPE_IO_USER_DIR='~/.config/miniscope_io' +export MINISCOPE_IO_LOG_DIR='~/config/miniscope_io/logs' +export MINISCOPE_IO_LOGS__LEVEL_FILE='INFO' +export MINISCOPE_IO_LOGS__LEVEL_STREAM='WARNING' +export MINISCOPE_IO_LOGS__FILE_N=5 +``` +```` +````{tab-item} .env file +```{code-block} python +MINISCOPE_IO_USER_DIR='~/.config/miniscope_io' +MINISCOPE_IO_LOG_DIR='~/config/miniscope_io/logs' +MINISCOPE_IO_LOG__LEVEL_FILE='INFO' +MINISCOPE_IO_LOG__LEVEL_STREAM='WARNING' +MINISCOPE_IO_LOG__FILE_N=5 +``` +```` +````{tab-item} pyproject.toml +```{code-block} toml +[tool.miniscope_io.config] +user_dir = "~/.config/miniscope_io" + +[tool.linkml.config.log] +dir = "~/config/miniscope_io/logs" +level_file = "INFO" +level_stream = "WARNING" +file_n = 5 +``` +```` +````{tab-item} cli +TODO +```` +````` + +## Device Configs + +```{todo} +Document device configuration +``` \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 6c83bc76..e4f112c7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -7,7 +7,8 @@ Generic I/O interfaces for miniscopes :) :maxdepth: 2 guide/installation -cli/main +guide/config +cli/index ``` ```{toctree} diff --git a/miniscope_io/cli/config.py b/miniscope_io/cli/config.py new file mode 100644 index 00000000..2952bf8f --- /dev/null +++ b/miniscope_io/cli/config.py @@ -0,0 +1,148 @@ +""" +CLI commands for configuration +""" + +from pathlib import Path + +import click +import yaml + +from miniscope_io.models import config as _config +from miniscope_io.models.config import set_user_dir + + +@click.group(invoke_without_command=True) +@click.pass_context +def config(ctx: click.Context) -> None: + """ + Command group for config + + When run without arguments, displays current config from all sources + """ + if ctx.invoked_subcommand is None: + config_str = _config.Config().to_yaml() + click.echo(f"miniscope-io configuration:\n-----\n{config_str}") + + +@config.group("global", invoke_without_command=True) +@click.pass_context +def global_(ctx: click.Context) -> None: + """ + Command group for global configuration directory + + When run without arguments, displays contents of current global config + """ + if ctx.invoked_subcommand is None: + + with open(_config._global_config_path) as f: + config_str = f.read() + + click.echo(f"Global configuration: {str(_config._global_config_path)}\n-----\n{config_str}") + + +@global_.command("path") +def global_path() -> None: + """Location of the global miniscope-io config""" + click.echo(str(_config._global_config_path)) + + +@config.group(invoke_without_command=True) +@click.pass_context +def user(ctx: click.Context) -> None: + """ + Command group for the user config directory + + When invoked without arguments, displays the contents of the current user directory + """ + if ctx.invoked_subcommand is None: + config = _config.Config() + config_file = list(config.user_dir.glob("mio_config.*")) + if len(config_file) == 0: + click.echo( + f"User directory specified as {str(config.user_dir)} " + "but no mio_config.yaml file found" + ) + return + else: + config_file = config_file[0] + + with open(config_file) as f: + config_str = f.read() + + click.echo(f"User configuration: {str(config_file)}\n-----\n{config_str}") + + +@user.command("create") +@click.argument("user_dir", type=click.Path(), required=False) +@click.option( + "--force/--no-force", + default=False, + help="Overwrite existing config file if it exists", +) +@click.option( + "--clean/--dirty", + default=False, + help="Create a fresh mio_config.yaml file containing only the user_dir. " + "Otherwise, by default (--dirty), any other settings from .env, pyproject.toml, etc." + "are included in the created user config file.", +) +@click.option( + "--dry-run/--no-dry-run", + default=False, + help="Show the config that would be written and where it would go without doing anything", +) +def create( + user_dir: Path = None, force: bool = False, clean: bool = False, dry_run: bool = False +) -> None: + """ + Create a user directory, + setting it as the default in the global config + + Args: + user_dir (Path): Path to the directory to create + force (bool): Overwrite existing config file if it exists + """ + if user_dir is None: + user_dir = _config._default_userdir + + try: + user_dir = Path(user_dir).expanduser().resolve() + except RuntimeError: + user_dir = Path(user_dir).resolve() + + if user_dir.is_file and user_dir.suffix in (".yaml", ".yml"): + config_file = user_dir + user_dir = user_dir.parent + else: + config_file = user_dir / "mio_config.yaml" + + if config_file.exists() and not force and not dry_run: + click.echo(f"Config file already exists at {str(config_file)}, use --force to overwrite") + return + + if clean: + config = {"user_dir": str(user_dir)} + + if not dry_run: + with open(config_file, "w") as f: + yaml.safe_dump(config, f) + + config_str = yaml.safe_dump(config) + else: + config = _config.Config(user_dir=user_dir) + config_str = config.to_yaml() if dry_run else config.to_yaml(config_file) + + # update global config pointer + if not dry_run: + set_user_dir(user_dir) + + prefix = "DRY RUN - No files changed\n-----\nWould have created" if dry_run else "Created" + + click.echo(f"{prefix} user config at {str(config_file)}:\n-----\n{config_str}") + + +@user.command("path") +def user_path() -> None: + """Location of the current user config""" + path = list(_config.Config().user_dir.glob("mio_config.*"))[0] + click.echo(str(path)) diff --git a/miniscope_io/cli/main.py b/miniscope_io/cli/main.py index 68d0fabb..b23d20e2 100644 --- a/miniscope_io/cli/main.py +++ b/miniscope_io/cli/main.py @@ -4,6 +4,7 @@ import click +from miniscope_io.cli.config import config from miniscope_io.cli.stream import stream from miniscope_io.cli.update import device, update @@ -21,3 +22,4 @@ def cli(ctx: click.Context) -> None: cli.add_command(stream) cli.add_command(update) cli.add_command(device) +cli.add_command(config) diff --git a/miniscope_io/models/config.py b/miniscope_io/models/config.py index 51f096e8..c7fcb079 100644 --- a/miniscope_io/models/config.py +++ b/miniscope_io/models/config.py @@ -3,17 +3,102 @@ """ from pathlib import Path -from typing import Literal, Optional +from typing import Any, Literal, Optional -from pydantic import Field, field_validator, model_validator -from pydantic_settings import BaseSettings, SettingsConfigDict +import yaml +from platformdirs import PlatformDirs +from pydantic import Field, TypeAdapter, field_validator, model_validator +from pydantic_settings import ( + BaseSettings, + PydanticBaseSettingsSource, + PyprojectTomlConfigSettingsSource, + SettingsConfigDict, + YamlConfigSettingsSource, +) from miniscope_io.models import MiniscopeIOModel +from miniscope_io.models.mixins import YAMLMixin -_default_basedir = Path().home() / ".config" / "miniscope_io" +_default_userdir = Path().home() / ".config" / "miniscope_io" +_dirs = PlatformDirs("miniscope_io", "miniscope_io") +_global_config_path = Path(_dirs.user_config_path) / "mio_config.yaml" LOG_LEVELS = Literal["DEBUG", "INFO", "WARNING", "ERROR"] +def _create_default_global_config(path: Path = _global_config_path, force: bool = False) -> None: + """ + Create a default global `mio_config.yaml` file to point to the user directory, + returning it. + + Args: + force (bool): Override any existing global config + """ + if path.exists() and not force: + return + + path.parent.mkdir(parents=True, exist_ok=True) + config = {"user_dir": str(path.parent)} + with open(path, "w") as f: + yaml.safe_dump(config, f) + + +class _UserYamlConfigSource(YamlConfigSettingsSource): + """ + Yaml config source that gets the location of the user settings file from the prior sources + """ + + def __init__(self, *args: Any, **kwargs: Any): + self._user_config = None + super().__init__(*args, **kwargs) + + @property + def user_config_path(self) -> Optional[Path]: + """ + Location of the user-level ``mio_config.yaml`` file, + given the current state of prior config sources, + including the global config file + """ + config_file = None + user_dir: Optional[str] = self.current_state.get("user_dir", None) + if user_dir is None: + # try and get from global config + if _global_config_path.exists(): + with open(_global_config_path) as f: + data = yaml.safe_load(f) + user_dir = data.get("user_dir", None) + + if user_dir is not None: + # handle .yml or .yaml + config_files = list(Path(user_dir).glob("mio_config.*")) + if len(config_files) != 0: + config_file = config_files[0] + + else: + # gotten from higher priority config sources + config_file = Path(user_dir) / "mio_config.yaml" + return config_file + + @property + def user_config(self) -> dict[str, Any]: + """ + Contents of the user config file + """ + if self._user_config is None: + if self.user_config_path is None or not self.user_config_path.exists(): + self._user_config = {} + else: + self._user_config = self._read_files(self.user_config_path) + + return self._user_config + + def __call__(self) -> dict[str, Any]: + return ( + TypeAdapter(dict[str, Any]).dump_python(self.user_config) + if self.nested_model_default_partial_update + else self.user_config + ) + + class LogConfig(MiniscopeIOModel): """ Configuration for logging @@ -62,7 +147,7 @@ def inherit_base_level(self) -> "LogConfig": return self -class Config(BaseSettings): +class Config(BaseSettings, YAMLMixin): """ Runtime configuration for miniscope-io. @@ -74,23 +159,22 @@ class Config(BaseSettings): See ``.env.example`` in repository root - Paths are set relative to ``base_dir`` by default, unless explicitly specified. - - + Paths are set relative to ``user_dir`` by default, unless explicitly specified. """ - base_dir: Path = Field( - _default_basedir, - description="Base directory to store configuration and other temporary files, " + user_dir: Path = Field( + _global_config_path.parent, + description="Base directory to store user configuration and other temporary files, " "other paths are relative to this by default", ) log_dir: Path = Field(Path("logs"), description="Location to store logs") + devices_dir: Path = Field(Path("devices"), description="Location to store device configs") logs: LogConfig = Field(LogConfig(), description="Additional settings for logs") - @field_validator("base_dir", mode="before") + @field_validator("user_dir", mode="before") @classmethod def folder_exists(cls, v: Path) -> Path: - """Ensure base_dir exists, make it otherwise""" + """Ensure user_dir exists, make it otherwise""" v = Path(v) v.mkdir(exist_ok=True, parents=True) @@ -99,12 +183,12 @@ def folder_exists(cls, v: Path) -> Path: @model_validator(mode="after") def paths_relative_to_basedir(self) -> "Config": - """If relative paths are given, make them absolute relative to ``base_dir``""" - paths = ("log_dir",) + """If relative paths are given, make them absolute relative to ``user_dir``""" + paths = ("log_dir", "devices_dir") for path_name in paths: path = getattr(self, path_name) # type: Path if not path.is_absolute(): - path = self.base_dir / path + path = self.user_dir / path setattr(self, path_name, path) path.mkdir(exist_ok=True) assert path.exists() @@ -115,4 +199,72 @@ def paths_relative_to_basedir(self) -> "Config": env_file=".env", env_nested_delimiter="__", extra="ignore", + nested_model_default_partial_update=True, + yaml_file="mio_config.yaml", + pyproject_toml_table_header=("tool", "miniscope_io", "config"), ) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + """ + Read config settings from, in order of priority from high to low, where + high priorities override lower priorities: + * in the arguments passed to the class constructor (not user configurable) + * in environment variables like ``export MINISCOPE_IO_LOG_DIR=~/`` + * in a ``.env`` file in the working directory + * in a ``mio_config.yaml`` file in the working directory + * in the ``tool.miniscope_io.config`` table in a ``pyproject.toml`` file + in the working directory + * in a user ``mio_config.yaml`` file, configured by `user_dir` in any of the other sources + * in the global ``mio_config.yaml`` file in the platform-specific data directory + (use ``mio config get global_config`` to find its location) + * the default values in the :class:`.GlobalConfig` model + """ + _create_default_global_config() + + return ( + init_settings, + env_settings, + dotenv_settings, + YamlConfigSettingsSource(settings_cls), + PyprojectTomlConfigSettingsSource(settings_cls), + _UserYamlConfigSource(settings_cls), + YamlConfigSettingsSource(settings_cls, yaml_file=_global_config_path), + ) + + +def set_user_dir(path: Path) -> None: + """ + Set the location of the user dir in the global config file + """ + _update_value(_global_config_path, "user_dir", str(path)) + + +def _update_value(path: Path, key: str, value: Any) -> None: + """ + Update a single value in a yaml file + + .. todo:: + + Make this work with nested keys + + """ + data = None + if path.exists(): + with open(path) as f: + data = yaml.safe_load(f) + + if data is None: + data = {} + + data[key] = value + + with open(path, "w") as f: + yaml.dump(data, f) diff --git a/miniscope_io/models/mixins.py b/miniscope_io/models/mixins.py index 18723ce3..4d011fe7 100644 --- a/miniscope_io/models/mixins.py +++ b/miniscope_io/models/mixins.py @@ -4,13 +4,25 @@ """ from pathlib import Path -from typing import Type, TypeVar, Union +from typing import Any, Optional, Type, TypeVar, Union import yaml +from pydantic import BaseModel T = TypeVar("T") +class YamlDumper(yaml.SafeDumper): + """Dumper that can represent extra types like Paths""" + + def represent_path(self, data: Path) -> yaml.ScalarNode: + """Represent a path as a string""" + return self.represent_scalar("tag:yaml.org,2002:str", str(data)) + + +YamlDumper.add_representer(type(Path()), YamlDumper.represent_path) + + class YAMLMixin: """ Mixin class that provides :meth:`.from_yaml` and :meth:`.to_yaml` @@ -23,3 +35,18 @@ def from_yaml(cls: Type[T], file_path: Union[str, Path]) -> T: with open(file_path) as file: config_data = yaml.safe_load(file) return cls(**config_data) + + def to_yaml(self, path: Optional[Path] = None, **kwargs: Any) -> str: + """ + Dump the contents of this class to a yaml file, returning the + contents of the dumped string + """ + data = self.model_dump(**kwargs) if isinstance(self, BaseModel) else self.__dict__ + + data = yaml.dump(data, Dumper=YamlDumper) + + if path: + with open(path, "w") as file: + file.write(data) + + return data diff --git a/pdm.lock b/pdm.lock index 3008efc3..695a578d 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "all", "dev", "docs", "plot", "tests"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:bf968fd0a212fae996b67713a6df6df51a9f5751b2493341be298900fba9cb42" +content_hash = "sha256:9d37dabdbf19b85da54f6640b090359128c49a1bc4a5d802968772fcb7854411" [[metadata.targets]] requires_python = "~=3.9" @@ -26,7 +26,7 @@ name = "annotated-types" version = "0.7.0" requires_python = ">=3.8" summary = "Reusable constraint types to use with typing.Annotated" -groups = ["default", "all", "docs"] +groups = ["default", "all", "docs", "tests"] dependencies = [ "typing-extensions>=4.0.0; python_version < \"3.9\"", ] @@ -83,7 +83,7 @@ files = [ name = "bitarray" version = "2.9.3" summary = "efficient arrays of booleans -- C extension" -groups = ["default"] +groups = ["default", "tests"] files = [ {file = "bitarray-2.9.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2cf5f5400636c7dda797fd681795ce63932458620fe8c40955890380acba9f62"}, {file = "bitarray-2.9.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3487b4718ffa5942fab777835ee36085f8dda7ec4bd0b28433efb117f84852b6"}, @@ -178,7 +178,7 @@ name = "bitstring" version = "4.2.3" requires_python = ">=3.8" summary = "Simple construction, analysis and modification of binary data." -groups = ["default"] +groups = ["default", "tests"] dependencies = [ "bitarray<3.0.0,>=2.9.0", ] @@ -340,7 +340,7 @@ name = "click" version = "8.1.7" requires_python = ">=3.7" summary = "Composable command line interface toolkit" -groups = ["default", "all", "dev", "docs"] +groups = ["default", "all", "dev", "docs", "tests"] dependencies = [ "colorama; platform_system == \"Windows\"", "importlib-metadata; python_version < \"3.8\"", @@ -903,7 +903,7 @@ name = "markdown-it-py" version = "3.0.0" requires_python = ">=3.8" summary = "Python port of markdown-it. Markdown parsing, done right!" -groups = ["default", "all", "docs"] +groups = ["default", "all", "docs", "tests"] dependencies = [ "mdurl~=0.1", ] @@ -1062,7 +1062,7 @@ name = "mdurl" version = "0.1.2" requires_python = ">=3.7" summary = "Markdown URL utilities" -groups = ["default", "all", "docs"] +groups = ["default", "all", "docs", "tests"] files = [ {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, @@ -1168,7 +1168,7 @@ name = "opencv-python" version = "4.10.0.84" requires_python = ">=3.6" summary = "Wrapper package for OpenCV python bindings." -groups = ["default"] +groups = ["default", "tests"] dependencies = [ "numpy>=1.13.3; python_version < \"3.7\"", "numpy>=1.17.0; python_version >= \"3.7\"", @@ -1207,7 +1207,7 @@ name = "pandas" version = "2.2.3" requires_python = ">=3.9" summary = "Powerful data structures for data analysis, time series, and statistics" -groups = ["default"] +groups = ["default", "tests"] dependencies = [ "numpy>=1.22.4; python_version < \"3.11\"", "numpy>=1.23.2; python_version == \"3.11\"", @@ -1361,7 +1361,7 @@ name = "platformdirs" version = "4.3.6" requires_python = ">=3.8" summary = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." -groups = ["all", "dev"] +groups = ["default", "all", "dev", "tests"] files = [ {file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"}, {file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"}, @@ -1401,7 +1401,7 @@ name = "pydantic" version = "2.9.2" requires_python = ">=3.8" summary = "Data validation using Python type hints" -groups = ["default", "all", "docs"] +groups = ["default", "all", "docs", "tests"] dependencies = [ "annotated-types>=0.6.0", "pydantic-core==2.23.4", @@ -1418,7 +1418,7 @@ name = "pydantic-core" version = "2.23.4" requires_python = ">=3.8" summary = "Core functionality for Pydantic validation and serialization" -groups = ["default", "all", "docs"] +groups = ["default", "all", "docs", "tests"] dependencies = [ "typing-extensions!=4.7.0,>=4.6.0", ] @@ -1507,7 +1507,7 @@ name = "pydantic-settings" version = "2.6.1" requires_python = ">=3.8" summary = "Settings management using Pydantic" -groups = ["default", "all", "docs"] +groups = ["default", "all", "docs", "tests"] dependencies = [ "pydantic>=2.7.0", "python-dotenv>=0.21.0", @@ -1522,7 +1522,7 @@ name = "pygments" version = "2.18.0" requires_python = ">=3.8" summary = "Pygments is a syntax highlighting package written in Python." -groups = ["default", "all", "docs"] +groups = ["default", "all", "docs", "tests"] files = [ {file = "pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"}, {file = "pygments-2.18.0.tar.gz", hash = "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199"}, @@ -1543,7 +1543,7 @@ files = [ name = "pyserial" version = "3.5" summary = "Python Serial Port Extension" -groups = ["default"] +groups = ["default", "tests"] files = [ {file = "pyserial-3.5-py2.py3-none-any.whl", hash = "sha256:c4451db6ba391ca6ca299fb3ec7bae67a5c55dde170964c7a14ceefec02f2cf0"}, {file = "pyserial-3.5.tar.gz", hash = "sha256:3c77e014170dfffbd816e6ffc205e9842efb10be9f58ec16d3e8675b4925cddb"}, @@ -1616,7 +1616,7 @@ name = "python-dotenv" version = "1.0.1" requires_python = ">=3.8" summary = "Read key-value pairs from a .env file and set them as environment variables" -groups = ["default", "all", "docs"] +groups = ["default", "all", "docs", "tests"] files = [ {file = "python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca"}, {file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"}, @@ -1626,7 +1626,7 @@ files = [ name = "pytz" version = "2024.2" summary = "World timezone definitions, modern and historical" -groups = ["default"] +groups = ["default", "tests"] files = [ {file = "pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"}, {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"}, @@ -1637,7 +1637,7 @@ name = "pyyaml" version = "6.0.2" requires_python = ">=3.8" summary = "YAML parser and emitter for Python" -groups = ["default", "all", "dev", "docs"] +groups = ["default", "all", "dev", "docs", "tests"] files = [ {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, @@ -1709,7 +1709,7 @@ name = "rich" version = "13.9.4" requires_python = ">=3.8.0" summary = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -groups = ["default"] +groups = ["default", "tests"] dependencies = [ "markdown-it-py>=2.2.0", "pygments<3.0.0,>=2.13.0", @@ -1840,6 +1840,20 @@ files = [ {file = "sphinx_click-6.0.0.tar.gz", hash = "sha256:f5d664321dc0c6622ff019f1e1c84e58ce0cecfddeb510e004cf60c2a3ab465b"}, ] +[[package]] +name = "sphinx-design" +version = "0.6.1" +requires_python = ">=3.9" +summary = "A sphinx extension for designing beautiful, view size responsive web components." +groups = ["all", "docs"] +dependencies = [ + "sphinx<9,>=6", +] +files = [ + {file = "sphinx_design-0.6.1-py3-none-any.whl", hash = "sha256:b11f37db1a802a183d61b159d9a202314d4d2fe29c163437001324fe2f19549c"}, + {file = "sphinx_design-0.6.1.tar.gz", hash = "sha256:b44eea3719386d04d765c1a8257caca2b3e6f8421d7b3a5e742c0fd45f84e632"}, +] + [[package]] name = "sphinxcontrib-applehelp" version = "2.0.0" @@ -1884,6 +1898,20 @@ files = [ {file = "sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178"}, ] +[[package]] +name = "sphinxcontrib-programoutput" +version = "0.17" +requires_python = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" +summary = "Sphinx extension to include program output" +groups = ["all", "docs"] +dependencies = [ + "Sphinx>=1.7.0", +] +files = [ + {file = "sphinxcontrib-programoutput-0.17.tar.gz", hash = "sha256:300ee9b8caee8355d25cc74b4d1c7efd12e608d2ad165e3141d31e6fbc152b7f"}, + {file = "sphinxcontrib_programoutput-0.17-py2.py3-none-any.whl", hash = "sha256:0ef1c1d9159dbe7103077748214305eb4e0138e861feb71c0c346afc5fe97f84"}, +] + [[package]] name = "sphinxcontrib-qthelp" version = "2.0.0" @@ -1918,12 +1946,23 @@ files = [ {file = "tomli-2.0.2.tar.gz", hash = "sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed"}, ] +[[package]] +name = "tomli-w" +version = "1.1.0" +requires_python = ">=3.9" +summary = "A lil' TOML writer" +groups = ["all", "tests"] +files = [ + {file = "tomli_w-1.1.0-py3-none-any.whl", hash = "sha256:1403179c78193e3184bfaade390ddbd071cba48a32a2e62ba11aae47490c63f7"}, + {file = "tomli_w-1.1.0.tar.gz", hash = "sha256:49e847a3a304d516a169a601184932ef0f6b61623fe680f836a2aa7128ed0d33"}, +] + [[package]] name = "tqdm" version = "4.66.6" requires_python = ">=3.7" summary = "Fast, Extensible Progress Meter" -groups = ["default"] +groups = ["default", "tests"] dependencies = [ "colorama; platform_system == \"Windows\"", ] @@ -1937,7 +1976,7 @@ name = "typing-extensions" version = "4.12.2" requires_python = ">=3.8" summary = "Backported and Experimental Type Hints for Python 3.8+" -groups = ["default", "all", "dev", "docs"] +groups = ["default", "all", "dev", "docs", "tests"] files = [ {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, @@ -1948,7 +1987,7 @@ name = "tzdata" version = "2024.2" requires_python = ">=2" summary = "Provider of IANA time zone data" -groups = ["default"] +groups = ["default", "tests"] files = [ {file = "tzdata-2024.2-py2.py3-none-any.whl", hash = "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd"}, {file = "tzdata-2024.2.tar.gz", hash = "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc"}, diff --git a/pyproject.toml b/pyproject.toml index fd387b79..a1647a25 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ dependencies = [ "rich>=13.6.0", "pyyaml>=6.0.1", "click>=8.1.7", + "platformdirs>=4.3.6", ] readme = "README.md" @@ -59,6 +60,7 @@ tests = [ "pytest-cov>=5.0.0", "pytest-timeout>=2.3.1", "miniscope_io[plot]", + "tomli-w>=1.1.0", ] docs = [ "sphinx>=6.2.1", @@ -66,6 +68,8 @@ docs = [ "furo>2023.07.26", "myst-parser>3.0.0", "autodoc-pydantic>=2.0.1", + "sphinxcontrib-programoutput>=0.17", + "sphinx-design>=0.6.1", ] dev = [ "black>=24.1.1", diff --git a/tests/test_config.py b/tests/test_config.py index 3bd488d4..edae6e0b 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,35 +1,218 @@ import os +from collections.abc import MutableMapping from pathlib import Path +from typing import Any, Callable + +import pytest +import yaml +import numpy as np +import tomli_w + from miniscope_io import Config +from miniscope_io.models.config import _global_config_path, set_user_dir +from miniscope_io.models.mixins import YamlDumper + + +@pytest.fixture(scope="module", autouse=True) +def dodge_existing_configs(tmp_path_factory): + """ + Suspend any existing global config file during config tests + """ + tmp_path = tmp_path_factory.mktemp("config_backup") + global_config_path = _global_config_path + backup_global_config_path = tmp_path / "mio_config.yaml.global.bak" + + user_config_path = list(Config().user_dir.glob("mio_config.*")) + if len(user_config_path) == 0: + user_config_path = None + else: + user_config_path = user_config_path[0] + + backup_user_config_path = tmp_path / "mio_config.yaml.user.bak" + + dotenv_path = Path(".env").resolve() + dotenv_backup_path = tmp_path / "dotenv.bak" + + if global_config_path.exists(): + global_config_path.rename(backup_global_config_path) + if user_config_path is not None and user_config_path.exists(): + user_config_path.rename(backup_user_config_path) + if dotenv_path.exists(): + dotenv_path.rename(dotenv_backup_path) + + yield + + if backup_global_config_path.exists(): + global_config_path.unlink(missing_ok=True) + backup_global_config_path.rename(global_config_path) + if backup_user_config_path.exists(): + user_config_path.unlink(missing_ok=True) + backup_user_config_path.rename(user_config_path) + if dotenv_backup_path.exists(): + dotenv_path.unlink(missing_ok=True) + dotenv_backup_path.rename(dotenv_path) + + +@pytest.fixture() +def tmp_cwd(tmp_path, monkeypatch) -> Path: + monkeypatch.chdir(tmp_path) + return tmp_path + + +@pytest.fixture() +def set_env(monkeypatch) -> Callable[[dict[str, Any]], None]: + """ + Function fixture to set environment variables using a nested dict + matching a GlobalConfig.model_dump() + """ + + def _set_env(config: dict[str, Any]) -> None: + for key, value in _flatten(config).items(): + key = "MINISCOPE_IO_" + key.upper() + monkeypatch.setenv(key, str(value)) + + return _set_env + + +@pytest.fixture() +def set_dotenv(tmp_cwd) -> Callable[[dict[str, Any]], Path]: + """ + Function fixture to set config variables in a .env file + """ + dotenv_path = tmp_cwd / ".env" + + def _set_dotenv(config: dict[str, Any]) -> Path: + with open(dotenv_path, "w") as dfile: + for key, value in _flatten(config).items(): + key = "MINISCOPE_IO_" + key.upper() + dfile.write(f"{key}={value}\n") + return dotenv_path + + return _set_dotenv + + +@pytest.fixture() +def set_pyproject(tmp_cwd) -> Callable[[dict[str, Any]], Path]: + """ + Function fixture to set config variables in a pyproject.toml file + """ + toml_path = tmp_cwd / "pyproject.toml" + + def _set_pyproject(config: dict[str, Any]) -> Path: + config = {"tool": {"miniscope_io": {"config": config}}} + + with open(toml_path, "wb") as tfile: + tomli_w.dump(config, tfile) + + return toml_path + + return _set_pyproject + + +@pytest.fixture() +def set_local_yaml(tmp_cwd) -> Callable[[dict[str, Any]], Path]: + """ + Function fixture to set config variables in a mio_config.yaml file in the current directory + """ + yaml_path = tmp_cwd / "mio_config.yaml" + + def _set_local_yaml(config: dict[str, Any]) -> Path: + with open(yaml_path, "w") as yfile: + yaml.dump(config, yfile, Dumper=YamlDumper) + return yaml_path + + return _set_local_yaml + + +@pytest.fixture() +def set_user_yaml(tmp_path) -> Callable[[dict[str, Any]], Path]: + """ + Function fixture to set config variables in a user config file + """ + yaml_path = tmp_path / "user" / "mio_config.yaml" + yaml_path.parent.mkdir(exist_ok=True) + + def _set_user_yaml(config: dict[str, Any]) -> Path: + with open(yaml_path, "w") as yfile: + yaml.dump(config, yfile, Dumper=YamlDumper) + set_user_dir(yaml_path.parent) + return yaml_path + + yield _set_user_yaml + + _global_config_path.unlink(missing_ok=True) + + +@pytest.fixture() +def set_global_yaml() -> Callable[[dict[str, Any]], Path]: + """ + Function fixture to reversibly set config variables in a global mio_config.yaml file + """ + + def _set_global_yaml(config: dict[str, Any]) -> Path: + with open(_global_config_path, "w") as gfile: + yaml.dump(config, gfile, Dumper=YamlDumper) + return _global_config_path + + yield _set_global_yaml + + _global_config_path.unlink(missing_ok=True) + + +@pytest.fixture( + params=[ + "set_env", + "set_dotenv", + "set_pyproject", + "set_local_yaml", + "set_user_yaml", + "set_global_yaml", + ] +) +def set_config(request) -> Callable[[dict[str, Any]], Path]: + return request.getfixturevalue(request.param) + def test_config(tmp_path): """ Config should be able to make directories and set sensible defaults """ - config = Config(base_dir = tmp_path) - assert config.base_dir.exists() + config = Config(user_dir=tmp_path) + assert config.user_dir.exists() assert config.log_dir.exists() - assert config.log_dir == config.base_dir / 'logs' + assert config.log_dir == config.user_dir / "logs" + + +def test_set_config(set_config, tmp_path): + """We should be able to set parameters from all available modalities""" + file_n = int(np.random.default_rng().integers(0, 100)) + user_dir = tmp_path / f"fake/dir/{np.random.default_rng().integers(0, 100)}" + + set_config({"user_dir": str(user_dir), "logs": {"file_n": file_n}}) + + config = Config() + assert config.user_dir == user_dir + assert config.logs.file_n == file_n def test_config_from_environment(tmp_path): """ Setting environmental variables should set the config, including recursive models """ - os.environ['MINISCOPE_IO_BASE_DIR'] = str(tmp_path) + os.environ["MINISCOPE_IO_USER_DIR"] = str(tmp_path) # we can also override the default log dir name - override_logdir = Path(tmp_path) / 'fancylogdir' - os.environ['MINISCOPE_IO_LOG_DIR'] = str(override_logdir) + override_logdir = Path(tmp_path) / "fancylogdir" + os.environ["MINISCOPE_IO_LOG_DIR"] = str(override_logdir) # and also recursive models - os.environ['MINISCOPE_IO_LOGS__LEVEL'] = 'error' + os.environ["MINISCOPE_IO_LOGS__LEVEL"] = "error" config = Config() - assert config.base_dir == Path(tmp_path) + assert config.user_dir == Path(tmp_path) assert config.log_dir == override_logdir - assert config.logs.level == 'error'.upper() - del os.environ['MINISCOPE_IO_BASE_DIR'] - del os.environ['MINISCOPE_IO_LOG_DIR'] - del os.environ['MINISCOPE_IO_LOGS__LEVEL'] + assert config.logs.level == "error".upper() + del os.environ["MINISCOPE_IO_USER_DIR"] + del os.environ["MINISCOPE_IO_LOG_DIR"] + del os.environ["MINISCOPE_IO_LOGS__LEVEL"] def test_config_from_dotenv(tmp_path): @@ -38,10 +221,64 @@ def test_config_from_dotenv(tmp_path): this test can be more relaxed since its basically a repetition of previous """ - tmp_path.mkdir(exist_ok=True,parents=True) - dotenv = tmp_path / '.env' - with open(dotenv, 'w') as denvfile: - denvfile.write(f'MINISCOPE_IO_BASE_DIR={str(tmp_path)}') + tmp_path.mkdir(exist_ok=True, parents=True) + dotenv = tmp_path / ".env" + with open(dotenv, "w") as denvfile: + denvfile.write(f"MINISCOPE_IO_USER_DIR={str(tmp_path)}") + + config = Config(_env_file=dotenv, _env_file_encoding="utf-8") + assert config.user_dir == Path(tmp_path) + + +def test_set_user_dir(tmp_path): + """ + We should be able to set the user dir and the global config should respect it + """ + user_config = tmp_path / "mio_config.yml" + file_n = int(np.random.default_rng().integers(0, 100)) + with open(user_config, "w") as yfile: + yaml.dump({"logs": {"file_n": file_n}}, yfile) + + set_user_dir(tmp_path) + + with open(_global_config_path, "r") as gfile: + global_config = yaml.safe_load(gfile) + + assert global_config["user_dir"] == str(tmp_path) + assert Config().user_dir == tmp_path + assert Config().logs.file_n == file_n + + # we do this manual cleanup here and not in a fixture because we are testing + # that the things we are doing in the fixtures are working correctly! + _global_config_path.unlink(missing_ok=True) + + +def test_config_sources_overrides( + set_env, set_dotenv, set_pyproject, set_local_yaml, set_user_yaml, set_global_yaml +): + """Test that the different config sources are overridden in the correct order""" + set_global_yaml({"logs": {"file_n": 0}}) + assert Config().logs.file_n == 0 + set_user_yaml({"logs": {"file_n": 1}}) + assert Config().logs.file_n == 1 + set_pyproject({"logs": {"file_n": 2}}) + assert Config().logs.file_n == 2 + set_local_yaml({"logs": {"file_n": 3}}) + assert Config().logs.file_n == 3 + set_dotenv({"logs": {"file_n": 4}}) + assert Config().logs.file_n == 4 + set_env({"logs": {"file_n": 5}}) + assert Config().logs.file_n == 5 + assert Config(**{"logs": {"file_n": 6}}).logs.file_n == 6 + - config = Config(_env_file=dotenv, _env_file_encoding='utf-8') - assert config.base_dir == Path(tmp_path) +def _flatten(d, parent_key="", separator="__") -> dict: + """https://stackoverflow.com/a/6027615/13113166""" + items = [] + for key, value in d.items(): + new_key = parent_key + separator + key if parent_key else key + if isinstance(value, MutableMapping): + items.extend(_flatten(value, new_key, separator=separator).items()) + else: + items.append((new_key, value)) + return dict(items) diff --git a/tests/test_logging.py b/tests/test_logging.py index b528163f..5d435366 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -7,6 +7,7 @@ from miniscope_io.logging import init_logger + def test_init_logger(capsys, tmp_path): """ We should be able to @@ -14,33 +15,26 @@ def test_init_logger(capsys, tmp_path): - with separable levels """ - log_dir = Path(tmp_path) / 'logs' + log_dir = Path(tmp_path) / "logs" log_dir.mkdir() - log_file = log_dir / 'miniscope_io.test_logger.log' - logger = init_logger( - name='test_logger', - log_dir=log_dir, - level='INFO', - file_level='WARNING' - ) - warn_msg = 'Both loggers should show' + log_file = log_dir / "miniscope_io.test_logger.log" + logger = init_logger(name="test_logger", log_dir=log_dir, level="INFO", file_level="WARNING") + warn_msg = "Both loggers should show" logger.warning(warn_msg) # can't test for presence of string because logger can split lines depending on size of console # but there should be one WARNING in stdout captured = capsys.readouterr() - assert 'WARNING' in captured.out + assert "WARNING" in captured.out - with open(log_file, 'r') as lfile: + with open(log_file, "r") as lfile: log_str = lfile.read() - assert 'WARNING' in log_str + assert "WARNING" in log_str info_msg = "Now only stdout should show" logger.info(info_msg) captured = capsys.readouterr() - assert 'INFO' in captured.out - with open(log_file, 'r') as lfile: + assert "INFO" in captured.out + with open(log_file, "r") as lfile: log_str = lfile.read() - assert 'INFO' not in log_str - - + assert "INFO" not in log_str