diff --git a/docs/api/formats/index.md b/docs/api/formats/index.md deleted file mode 100644 index 0f181006..00000000 --- a/docs/api/formats/index.md +++ /dev/null @@ -1,12 +0,0 @@ -# formats - -```{eval-rst} -.. automodule:: miniscope_io.formats - :members: - :undoc-members: -``` - -```{toctree} -sdcard -stream -``` \ No newline at end of file diff --git a/docs/api/formats/sdcard.md b/docs/api/formats/sdcard.md deleted file mode 100644 index 60971b8a..00000000 --- a/docs/api/formats/sdcard.md +++ /dev/null @@ -1,7 +0,0 @@ -# sdcard - -```{eval-rst} -.. automodule:: miniscope_io.formats.sdcard - :members: - :undoc-members: -``` \ No newline at end of file diff --git a/docs/api/formats/stream.md b/docs/api/formats/stream.md deleted file mode 100644 index 9cca38d0..00000000 --- a/docs/api/formats/stream.md +++ /dev/null @@ -1,7 +0,0 @@ -# stream - -```{eval-rst} -.. automodule:: miniscope_io.formats.stream - :members: - :undoc-members: -``` \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 6c83bc76..18a507c0 100644 --- a/docs/index.md +++ b/docs/index.md @@ -23,7 +23,6 @@ device/update_controller :caption: API: api/devices -api/formats/index api/io api/logging api/models/index diff --git a/miniscope_io/__init__.py b/miniscope_io/__init__.py index 99cf0494..f017b217 100644 --- a/miniscope_io/__init__.py +++ b/miniscope_io/__init__.py @@ -5,7 +5,6 @@ from importlib import metadata from pathlib import Path -from miniscope_io.io import SDCard from miniscope_io.logging import init_logger from miniscope_io.models.config import Config @@ -19,7 +18,6 @@ "DATA_DIR", "CONFIG_DIR", "Config", - "SDCard", "init_logger", ] diff --git a/miniscope_io/cli/common.py b/miniscope_io/cli/common.py new file mode 100644 index 00000000..0d09f36e --- /dev/null +++ b/miniscope_io/cli/common.py @@ -0,0 +1,31 @@ +""" +Shared CLI utils +""" + +from os import PathLike +from pathlib import Path +from typing import Optional + +from click import Context, Parameter, ParamType + + +class ConfigIDOrPath(ParamType): + """ + A custom click type to accept either a config `id` or a path + as input, resolving relative paths first against + the current working directory and second against the user config directory. + """ + + name = "config-id-or-path" + + def convert( + self, value: str | PathLike[str], param: Optional[Parameter], ctx: Optional[Context] + ) -> str | Path: + """ + If something looks like a yaml file, return as a path, otherwise return unchanged. + + Don't do validation here, the Config model will handle that on instantiation. + """ + if value.endswith(".yaml") or value.endswith(".yml"): + value = Path(value) + return value diff --git a/miniscope_io/cli/stream.py b/miniscope_io/cli/stream.py index eb9af453..fcb0ced7 100644 --- a/miniscope_io/cli/stream.py +++ b/miniscope_io/cli/stream.py @@ -8,6 +8,7 @@ import click +from miniscope_io.cli.common import ConfigIDOrPath from miniscope_io.stream_daq import StreamDaq @@ -24,8 +25,13 @@ def _common_options(fn: Callable) -> Callable: "-c", "--device_config", required=True, - help="Path to device config YAML file for streamDaq (see models.stream.StreamDevConfig)", - type=click.Path(exists=True), + help=( + "Either a config `id` or a path to device config YAML file for streamDaq " + "(see models.stream.StreamDevConfig). If path is relative, treated as " + "relative to the current directory, and then if no matching file is found, " + "relative to the user `config_dir` (see `mio config --help`)." + ), + type=ConfigIDOrPath(), )(fn) return fn diff --git a/miniscope_io/data/config/wirefree/sd-layout-battery.yaml b/miniscope_io/data/config/wirefree/sd-layout-battery.yaml new file mode 100644 index 00000000..fc6e6c0e --- /dev/null +++ b/miniscope_io/data/config/wirefree/sd-layout-battery.yaml @@ -0,0 +1,41 @@ +id: wirefree-sd-layout-battery +mio_model: miniscope_io.models.sdcard.SDLayout +mio_version: v5.0.0 +sectors: + header: 1022 + config: 1023 + data: 1024 + size: 512 +write_key0: 226277911 +write_key1: 226277911 +write_key2: 226277911 +write_key3: 226277911 +word_size: 4 +header: + gain: 4 + led: 5 + ewl: 6 + record_length: 7 + fs: 8 + delay_start: 9 + battery_cutoff: 10 +config: + width: 0 + height: 1 + fs: 2 + buffer_size: 3 + n_buffers_recorded: 4 + n_buffers_dropped: 5 +buffer: + linked_list: 1 + frame_num: 2 + buffer_count: 3 + frame_buffer_count: 4 + write_buffer_count: 5 + dropped_buffer_count: 6 + timestamp: 7 + write_timestamp: 9 + length: 0 + data_length: 8 + battery_voltage: 10 +version: 0.1.1 diff --git a/miniscope_io/data/config/wirefree/sd-layout.yaml b/miniscope_io/data/config/wirefree/sd-layout.yaml new file mode 100644 index 00000000..184ed572 --- /dev/null +++ b/miniscope_io/data/config/wirefree/sd-layout.yaml @@ -0,0 +1,40 @@ +id: wirefree-sd-layout +mio_model: miniscope_io.models.sdcard.SDLayout +mio_version: v5.0.0 +sectors: + header: 1022 + config: 1023 + data: 1024 + size: 512 +write_key0: 226277911 +write_key1: 226277911 +write_key2: 226277911 +write_key3: 226277911 +word_size: 4 +header: + gain: 4 + led: 5 + ewl: 6 + record_length: 7 + fs: 8 + delay_start: 9 + battery_cutoff: 10 +config: + width: 0 + height: 1 + fs: 2 + buffer_size: 3 + n_buffers_recorded: 4 + n_buffers_dropped: 5 +buffer: + linked_list: 1 + frame_num: 2 + buffer_count: 3 + frame_buffer_count: 4 + write_buffer_count: 5 + dropped_buffer_count: 6 + timestamp: 7 + write_timestamp: null + length: 0 + data_length: 8 + battery_voltage: null diff --git a/miniscope_io/data/config/wireless/stream-buffer-header.yaml b/miniscope_io/data/config/wireless/stream-buffer-header.yaml new file mode 100644 index 00000000..80dc5cb0 --- /dev/null +++ b/miniscope_io/data/config/wireless/stream-buffer-header.yaml @@ -0,0 +1,14 @@ +id: stream-buffer-header +mio_model: miniscope_io.models.stream.StreamBufferHeaderFormat +mio_version: "v5.0.0" +linked_list: 0 +frame_num: 1 +buffer_count: 2 +frame_buffer_count: 3 +write_buffer_count: 4 +dropped_buffer_count: 5 +timestamp: 6 +write_timestamp: 8 +pixel_count: 7 +battery_voltage_raw: 9 +input_voltage_raw: 10 diff --git a/miniscope_io/data/config/WLMS_v02_200px.yml b/miniscope_io/data/config/wireless/wireless-200px.yml similarity index 92% rename from miniscope_io/data/config/WLMS_v02_200px.yml rename to miniscope_io/data/config/wireless/wireless-200px.yml index e5ef3cff..00f15c10 100644 --- a/miniscope_io/data/config/WLMS_v02_200px.yml +++ b/miniscope_io/data/config/wireless/wireless-200px.yml @@ -1,3 +1,7 @@ +id: wireless-200px +mio_model: miniscope_io.models.stream.StreamDevConfig +mio_version: "v5.0.0" + # capture device. "OK" (Opal Kelly) or "UART" device: "OK" diff --git a/miniscope_io/formats/__init__.py b/miniscope_io/formats/__init__.py deleted file mode 100644 index 5ab7fe30..00000000 --- a/miniscope_io/formats/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -""" -Instantiations of :class:`~.miniscope_io.models.MiniscopeConfig` models -that describe fixed per-device configurations for the generic config -models in :mod:`~.miniscope_io.models.stream` et al. -""" - -from miniscope_io.formats.sdcard import WireFreeSDLayout, WireFreeSDLayout_Battery - -__all__ = ["WireFreeSDLayout", "WireFreeSDLayout_Battery"] diff --git a/miniscope_io/formats/sdcard.py b/miniscope_io/formats/sdcard.py deleted file mode 100644 index 04dffc14..00000000 --- a/miniscope_io/formats/sdcard.py +++ /dev/null @@ -1,80 +0,0 @@ -""" -SD Card data layout formats for different miniscopes! -""" - -from miniscope_io.models.sdcard import ( - ConfigPositions, - SDBufferHeaderFormat, - SDHeaderPositions, - SDLayout, - SectorConfig, -) - -WireFreeSDLayout = SDLayout( - version="0.1.1", - sectors=SectorConfig(header=1022, config=1023, data=1024, size=512), - write_key0=0x0D7CBA17, - write_key1=0x0D7CBA17, - write_key2=0x0D7CBA17, - write_key3=0x0D7CBA17, - header=SDHeaderPositions( - gain=4, led=5, ewl=6, record_length=7, fs=8, delay_start=9, battery_cutoff=10 - ), - config=ConfigPositions( - width=0, - height=1, - fs=2, - buffer_size=3, - n_buffers_recorded=4, - n_buffers_dropped=5, - ), - buffer=SDBufferHeaderFormat( - length=0, - linked_list=1, - frame_num=2, - buffer_count=3, - frame_buffer_count=4, - write_buffer_count=5, - dropped_buffer_count=6, - timestamp=7, - data_length=8, - ), -) - -WireFreeSDLayout_Battery = SDLayout(**WireFreeSDLayout.model_dump()) -""" -Making another format for now, but added version field so that we could -replace making more top-level classes with a FormatCollection that can store -sets of formats for the same device with multiple versions. -""" -WireFreeSDLayout_Battery.buffer.write_timestamp = 9 -WireFreeSDLayout_Battery.buffer.battery_voltage = 10 - - -WireFreeSDLayout_Old = SDLayout( - sectors=SectorConfig(header=1023, config=1024, data=1025, size=512), - write_key0=0x0D7CBA17, - write_key1=0x0D7CBA17, - write_key2=0x0D7CBA17, - write_key3=0x0D7CBA17, - header=SDHeaderPositions(gain=4, led=5, ewl=6, record_length=7, fs=8), - config=ConfigPositions( - width=0, - height=1, - fs=2, - buffer_size=3, - n_buffers_recorded=4, - n_buffers_dropped=5, - ), - buffer=SDBufferHeaderFormat( - length=0, - linked_list=1, - frame_num=2, - buffer_count=3, - frame_buffer_count=4, - write_buffer_count=5, - dropped_buffer_count=6, - timestamp=7, - data_length=8, - ), -) diff --git a/miniscope_io/formats/stream.py b/miniscope_io/formats/stream.py deleted file mode 100644 index 4786491e..00000000 --- a/miniscope_io/formats/stream.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -Formats for use with :mod:`miniscope_io.stream_daq` -We plan to re-define this soon so documentation will come after that. -""" - -from miniscope_io.models.stream import StreamBufferHeaderFormat - -StreamBufferHeader = StreamBufferHeaderFormat( - linked_list=0, - frame_num=1, - buffer_count=2, - frame_buffer_count=3, - write_buffer_count=4, - dropped_buffer_count=5, - timestamp=6, - pixel_count=7, - write_timestamp=8, - battery_voltage_raw=9, - input_voltage_raw=10, -) diff --git a/miniscope_io/io.py b/miniscope_io/io.py index 3b902e94..dac61fe5 100644 --- a/miniscope_io/io.py +++ b/miniscope_io/io.py @@ -16,6 +16,7 @@ from miniscope_io.logging import init_logger from miniscope_io.models.data import Frame from miniscope_io.models.sdcard import SDBufferHeader, SDConfig, SDLayout +from miniscope_io.types import ConfigSource class BufferedCSVWriter: @@ -104,9 +105,11 @@ class SDCard: """ - def __init__(self, drive: Union[str, Path], layout: SDLayout): + def __init__( + self, drive: Union[str, Path], layout: Union[SDLayout, ConfigSource] = "wirefree-sd-layout" + ): self.drive = drive - self.layout = layout + self.layout = SDLayout.from_any(layout) self.logger = init_logger("SDCard") # Private attributes used when the file reading context is entered diff --git a/miniscope_io/models/buffer.py b/miniscope_io/models/buffer.py index 49b177fc..82f455b2 100644 --- a/miniscope_io/models/buffer.py +++ b/miniscope_io/models/buffer.py @@ -7,9 +7,10 @@ from typing import Type, TypeVar from miniscope_io.models import Container, MiniscopeConfig +from miniscope_io.models.mixins import ConfigYAMLMixin -class BufferHeaderFormat(MiniscopeConfig): +class BufferHeaderFormat(MiniscopeConfig, ConfigYAMLMixin): """ Format model used to parse header at the beginning of every buffer. @@ -86,7 +87,7 @@ def from_format( """ header_data = dict() - for hd, header_index in format.model_dump().items(): + for hd, header_index in format.model_dump(exclude=set(format.HEADER_FIELDS)).items(): if header_index is not None: header_data[hd] = vals[header_index] diff --git a/miniscope_io/models/config.py b/miniscope_io/models/config.py index 51f096e8..d33d6131 100644 --- a/miniscope_io/models/config.py +++ b/miniscope_io/models/config.py @@ -84,7 +84,9 @@ class Config(BaseSettings): description="Base directory to store configuration and other temporary files, " "other paths are relative to this by default", ) + config_dir: Path = Field(Path("config"), description="Location to store user configs") log_dir: Path = Field(Path("logs"), description="Location to store logs") + logs: LogConfig = Field(LogConfig(), description="Additional settings for logs") @field_validator("base_dir", mode="before") @@ -100,7 +102,7 @@ def folder_exists(cls, v: Path) -> Path: @model_validator(mode="after") def paths_relative_to_basedir(self) -> "Config": """If relative paths are given, make them absolute relative to ``base_dir``""" - paths = ("log_dir",) + paths = ("log_dir", "config_dir") for path_name in paths: path = getattr(self, path_name) # type: Path if not path.is_absolute(): diff --git a/miniscope_io/models/mixins.py b/miniscope_io/models/mixins.py index 18723ce3..091d238a 100644 --- a/miniscope_io/models/mixins.py +++ b/miniscope_io/models/mixins.py @@ -3,14 +3,34 @@ to use composition for functionality and inheritance for semantics. """ +import re +import shutil +from importlib.metadata import version +from itertools import chain from pathlib import Path -from typing import Type, TypeVar, Union +from typing import Any, ClassVar, List, Literal, Optional, Type, TypeVar, Union, overload import yaml +from pydantic import BaseModel, Field, ValidationError, field_validator + +from miniscope_io import CONFIG_DIR, Config +from miniscope_io.logging import init_logger +from miniscope_io.types import ConfigID, ConfigSource, PythonIdentifier, valid_config_id T = TypeVar("T") +class YamlDumper(yaml.SafeDumper): + """Dumper that can represent extra types like Paths""" + + def represent_path(self, data: Path) -> yaml.ScalarNode: + """Represent a path as a string""" + return self.represent_scalar("tag:yaml.org,2002:str", str(data)) + + +YamlDumper.add_representer(type(Path()), YamlDumper.represent_path) + + class YAMLMixin: """ Mixin class that provides :meth:`.from_yaml` and :meth:`.to_yaml` @@ -23,3 +43,273 @@ def from_yaml(cls: Type[T], file_path: Union[str, Path]) -> T: with open(file_path) as file: config_data = yaml.safe_load(file) return cls(**config_data) + + def to_yaml(self, path: Optional[Path] = None, **kwargs: Any) -> str: + """ + Dump the contents of this class to a yaml file, returning the + contents of the dumped string + """ + data_str = self.to_yamls(**kwargs) + if path: + with open(path, "w") as file: + file.write(data_str) + + return data_str + + def to_yamls(self, **kwargs: Any) -> str: + """ + Dump the contents of this class to a yaml string + + Args: + **kwargs: passed to :meth:`.BaseModel.model_dump` + """ + data = self._dump_data(**kwargs) + return yaml.dump(data, Dumper=YamlDumper, sort_keys=False) + + def _dump_data(self, **kwargs: Any) -> dict: + data = self.model_dump(**kwargs) if isinstance(self, BaseModel) else self.__dict__ + return data + + +class ConfigYAMLMixin(BaseModel, YAMLMixin): + """ + Yaml Mixin class that always puts a header consisting of + + * `id` - unique identifier for this config + * `mio_model` - fully-qualified module path to model class + * `mio_version` - version of miniscope-io when this model was created + + at the top of the file. + """ + + id: ConfigID + mio_model: PythonIdentifier = Field(None, validate_default=True) + mio_version: str = version("miniscope-io") + + HEADER_FIELDS: ClassVar[tuple[str]] = ("id", "mio_model", "mio_version") + + @classmethod + def from_yaml(cls: Type[T], file_path: Union[str, Path]) -> T: + """Instantiate this class by passing the contents of a yaml file as kwargs""" + with open(file_path) as file: + config_data = yaml.safe_load(file) + + # fill in any missing fields in the source file needed for a header + config_data = cls._complete_header(config_data, file_path) + try: + instance = cls(**config_data) + except ValidationError: + if (backup_path := file_path.with_suffix(".yaml.bak")).exists(): + init_logger("config").debug( + f"Model instantiation failed, restoring modified backup from {backup_path}..." + ) + shutil.copy(backup_path, file_path) + raise + + return instance + + @classmethod + def from_id(cls: Type[T], id: ConfigID) -> T: + """ + Instantiate a model from a config `id` specified in one of the .yaml configs in + either the user :attr:`.Config.config_dir` or the packaged ``config`` dir. + + .. note:: + + this method does not yet validate that the config matches the model loading it + + """ + globs = [src.rglob("*.y*ml") for src in cls.config_sources] + for config_file in chain(*globs): + try: + file_id = yaml_peek("id", config_file) + if file_id == id: + init_logger("config").debug( + "Model for %s found at %s", cls._model_name(), config_file + ) + return cls.from_yaml(config_file) + except KeyError: + continue + raise KeyError(f"No config with id {id} found in {Config().config_dir}") + + @classmethod + def from_any(cls: Type[T], source: Union[ConfigSource, T]) -> T: + """ + Try and instantiate a config model from any supported constructor. + + Args: + source (:class:`.ConfigID`, :class:`.Path`, :class:`.PathLike[str]`): + Either + + * the ``id`` of a config file in the user configs directory or builtin + * a relative ``Path`` to a config file, relative to the current working directory + * a relative ``Path`` to a config file, relative to the user config directory + * an absolute ``Path`` to a config file + * an instance of the class to be constructed (returned unchanged) + + """ + if isinstance(source, cls): + return source + elif valid_config_id(source): + return cls.from_id(source) + else: + source = Path(source) + if source.suffix in (".yaml", ".yml"): + if source.exists(): + # either relative to cwd or absolute + return cls.from_yaml(source) + elif ( + not source.is_absolute() + and (user_source := Config().config_dir / source).exists() + ): + return cls.from_yaml(user_source) + + raise ValueError( + f"Instance of config model {cls.__name__} could not be instantiated from " + f"{source} - id or file not found, or type not supported" + ) + + @field_validator("mio_model", mode="before") + @classmethod + def fill_mio_model(cls, v: Optional[str]) -> PythonIdentifier: + """Get name of instantiating model, if not provided""" + if v is None: + v = cls._model_name() + return v + + @classmethod + @property + def config_sources(cls: Type[T]) -> List[Path]: + """ + Directories to search for config files, in order of priority + such that earlier sources are preferred over later sources. + """ + return [Config().config_dir, CONFIG_DIR] + + def _dump_data(self, **kwargs: Any) -> dict: + """Ensure that header is prepended to model data""" + return {**self._yaml_header(self), **super()._dump_data(**kwargs)} + + @classmethod + def _model_name(cls) -> PythonIdentifier: + return f"{cls.__module__}.{cls.__name__}" + + @classmethod + def _yaml_header(cls, instance: Union[T, dict]) -> dict: + if isinstance(instance, dict): + model_id = instance.get("id", None) + mio_model = instance.get("mio_model", cls._model_name()) + mio_version = instance.get("mio_version", version("miniscope_io")) + else: + model_id = getattr(instance, "id", None) + mio_model = getattr(instance, "mio_model", cls._model_name()) + mio_version = getattr(instance, "mio_version", version("miniscope_io")) + + if model_id is None: + # if missing an id, try and recover with model default cautiously + # so we throw the exception during validation and not here, for clarity. + model_id = getattr(cls.model_fields.get("id", None), "default", None) + if type(model_id).__name__ == "PydanticUndefinedType": + model_id = None + + return { + "id": model_id, + "mio_model": mio_model, + "mio_version": mio_version, + } + + @classmethod + def _complete_header(cls: Type[T], data: dict, file_path: Union[str, Path]) -> dict: + """fill in any missing fields in the source file needed for a header""" + + missing_fields = set(cls.HEADER_FIELDS) - set(data.keys()) + keys = tuple(data.keys()) + out_of_order = len(keys) >= 3 and keys[0:3] != cls.HEADER_FIELDS + + if missing_fields or out_of_order: + if missing_fields: + msg = f"Missing required header fields {missing_fields} in config model " + f"{str(file_path)}. Updating file (preserving backup)..." + else: + msg = f"Header keys were present, but either not at the start of {str(file_path)} " + "or in out of order. Updating file (preserving backup)..." + logger = init_logger(cls.__name__) + logger.warning(msg) + logger.debug(data) + + header = cls._yaml_header(data) + data = {**header, **data} + if CONFIG_DIR not in file_path.parents: + shutil.copy(file_path, file_path.with_suffix(".yaml.bak")) + with open(file_path, "w") as yfile: + yaml.dump(data, yfile, Dumper=YamlDumper, sort_keys=False) + + return data + + +@overload +def yaml_peek( + key: str, path: Union[str, Path], root: bool = True, first: Literal[True] = True +) -> str: ... + + +@overload +def yaml_peek( + key: str, path: Union[str, Path], root: bool = True, first: Literal[False] = False +) -> List[str]: ... + + +@overload +def yaml_peek( + key: str, path: Union[str, Path], root: bool = True, first: bool = True +) -> Union[str, List[str]]: ... + + +def yaml_peek( + key: str, path: Union[str, Path], root: bool = True, first: bool = True +) -> Union[str, List[str]]: + """ + Peek into a yaml file without parsing the whole file to retrieve the value of a single key. + + This function is _not_ designed for robustness to the yaml spec, it is for simple key: value + pairs, not fancy shit like multiline strings, tagged values, etc. If you want it to be, + then i'm afraid you'll have to make a PR about it. + + Returns a string no matter what the yaml type is so ya have to do your own casting if you want + + Args: + key (str): The key to peek for + path (:class:`pathlib.Path` , str): The yaml file to peek into + root (bool): Only find keys at the root of the document (default ``True`` ), otherwise + find keys at any level of nesting. + first (bool): Only return the first appearance of the key (default). Otherwise return a + list of values (not implemented lol) + + Returns: + str + """ + if root: + pattern = re.compile( + rf"^(?P{key}):\s*\"*\'*(?P\S.*?)\"*\'*$", flags=re.MULTILINE + ) + else: + pattern = re.compile( + rf"^\s*(?P{key}):\s*\"*\'*(?P\S.*?)\"*\'*$", flags=re.MULTILINE + ) + + res = None + if first: + with open(path) as yfile: + for line in yfile: + res = pattern.match(line) + if res: + break + if res: + return res.groupdict()["value"] + else: + with open(path) as yfile: + text = yfile.read() + res = [match.groupdict()["value"] for match in pattern.finditer(text)] + if res: + return res + raise KeyError(f"Key {key} not found in {path}") diff --git a/miniscope_io/models/sdcard.py b/miniscope_io/models/sdcard.py index c6ad9395..04340931 100644 --- a/miniscope_io/models/sdcard.py +++ b/miniscope_io/models/sdcard.py @@ -8,6 +8,7 @@ from miniscope_io.models import MiniscopeConfig from miniscope_io.models.buffer import BufferHeader, BufferHeaderFormat +from miniscope_io.models.mixins import ConfigYAMLMixin class SectorConfig(MiniscopeConfig): @@ -91,6 +92,8 @@ class SDBufferHeaderFormat(BufferHeaderFormat): Positions in the header for each frame """ + id: str = "sd-buffer-header" + length: int = 0 linked_list: int = 1 frame_num: int = 2 @@ -104,7 +107,7 @@ class SDBufferHeaderFormat(BufferHeaderFormat): battery_voltage: Optional[int] = None -class SDLayout(MiniscopeConfig): +class SDLayout(MiniscopeConfig, ConfigYAMLMixin): """ Data layout of an SD Card. @@ -131,12 +134,6 @@ class SDLayout(MiniscopeConfig): config: ConfigPositions = ConfigPositions() buffer: SDBufferHeaderFormat = SDBufferHeaderFormat() - version: Optional[str] = None - """ - Not Implemented: version stored in the SD card header that indicates - when this layout should be used - """ - class SDConfig(MiniscopeConfig): """ diff --git a/miniscope_io/models/stream.py b/miniscope_io/models/stream.py index dbeaa1ca..69b88601 100644 --- a/miniscope_io/models/stream.py +++ b/miniscope_io/models/stream.py @@ -10,7 +10,7 @@ from miniscope_io import DEVICE_DIR from miniscope_io.models import MiniscopeConfig from miniscope_io.models.buffer import BufferHeader, BufferHeaderFormat -from miniscope_io.models.mixins import YAMLMixin +from miniscope_io.models.mixins import ConfigYAMLMixin from miniscope_io.models.sinks import CSVWriterConfig, StreamPlotterConfig @@ -168,7 +168,7 @@ class StreamDevRuntime(MiniscopeConfig): ) -class StreamDevConfig(MiniscopeConfig, YAMLMixin): +class StreamDevConfig(MiniscopeConfig, ConfigYAMLMixin): """ Format model used to parse DAQ configuration yaml file (examples are in ./config) The model attributes are key-value pairs needed for reconstructing frames from data streams. diff --git a/miniscope_io/stream_daq.py b/miniscope_io/stream_daq.py index 5caa0fb8..79d3e3ab 100644 --- a/miniscope_io/stream_daq.py +++ b/miniscope_io/stream_daq.py @@ -20,16 +20,14 @@ from miniscope_io.bit_operation import BufferFormatter from miniscope_io.devices.mocks import okDevMock from miniscope_io.exceptions import EndOfRecordingException, StreamReadError -from miniscope_io.formats.stream import StreamBufferHeader as StreamBufferHeaderFormat from miniscope_io.io import BufferedCSVWriter from miniscope_io.models.stream import ( StreamBufferHeader, + StreamBufferHeaderFormat, StreamDevConfig, ) -from miniscope_io.models.stream import ( - StreamBufferHeaderFormat as StreamBufferHeaderFormatType, -) from miniscope_io.plots.headers import StreamPlotter +from miniscope_io.types import ConfigSource HAVE_OK = False ok_error = None @@ -82,8 +80,8 @@ class StreamDaq: def __init__( self, - device_config: Union[StreamDevConfig, Path], - header_fmt: StreamBufferHeaderFormatType = StreamBufferHeaderFormat, + device_config: Union[StreamDevConfig, ConfigSource], + header_fmt: Union[StreamBufferHeaderFormat, ConfigSource] = "stream-buffer-header", ) -> None: """ Constructer for the class. @@ -100,12 +98,18 @@ def __init__( Header format used to parse information from buffer header, by default `MetadataHeaderFormat()`. """ - if isinstance(device_config, (str, Path)): - device_config = StreamDevConfig.from_yaml(device_config) self.logger = init_logger("streamDaq") - self.config = device_config - self.header_fmt = header_fmt + self.config = StreamDevConfig.from_any(device_config) + self.header_fmt = StreamBufferHeaderFormat.from_any(header_fmt) + if isinstance(header_fmt, str): + self.header_fmt = StreamBufferHeaderFormat.from_id(header_fmt) + elif isinstance(header_fmt, StreamBufferHeaderFormat): + self.header_fmt = header_fmt + else: + raise TypeError( + "header_fmt should be an instance of StreamBufferHeaderFormat or a config ID." + ) self.preamble = self.config.preamble self.terminate: multiprocessing.Event = multiprocessing.Event() diff --git a/miniscope_io/types.py b/miniscope_io/types.py index 32d31495..a24884b3 100644 --- a/miniscope_io/types.py +++ b/miniscope_io/types.py @@ -2,6 +2,65 @@ Type and type annotations """ -from typing import Tuple, Union +import re +import sys +from os import PathLike +from pathlib import Path +from typing import Annotated, Any, Tuple, Union -Range = Union[Tuple[int, int], Tuple[float, float]] +from pydantic import AfterValidator, Field + +if sys.version_info < (3, 10): + from typing_extensions import TypeAlias, TypeIs +elif sys.version_info < (3, 13): + from typing import TypeAlias + + from typing_extensions import TypeIs +else: + from typing import TypeAlias, TypeIs + +CONFIG_ID_PATTERN = r"[\w\-\/#]+" +""" +Any alphanumeric string (\w), as well as +- ``-`` +- ``/`` +- ``#`` +(to allow hierarchical IDs as well as fragment IDs). + +Specficially excludes ``.`` to avoid confusion between IDs, paths, and python module names + +May be made less restrictive in the future, will not be made more restrictive. +""" + + +def _is_identifier(val: str) -> str: + # private validation method to validate the parts of a fully-qualified python identifier + # defined first and not made public bc used as a validator, + # distinct from a boolean "is_{x}" check + + for part in val.split("."): + assert part.isidentifier(), f"{part} is not a valid python identifier within {val}" + return val + + +Range: TypeAlias = Union[Tuple[int, int], Tuple[float, float]] +PythonIdentifier: TypeAlias = Annotated[str, AfterValidator(_is_identifier)] +""" +A valid python identifier, including globally namespace pathed like +module.submodule.ClassName +""" +ConfigID: TypeAlias = Annotated[str, Field(pattern=CONFIG_ID_PATTERN)] +""" +A string that refers to a config file by the ``id`` field in that config +""" +ConfigSource: TypeAlias = Union[Path, PathLike[str], ConfigID] +""" +Union of all types of config sources +""" + + +def valid_config_id(val: Any) -> TypeIs[ConfigID]: + """ + Checks whether a string is a valid config id. + """ + return bool(re.fullmatch(CONFIG_ID_PATTERN, val)) diff --git a/notebooks/Wire-Free-DAQ.ipynb b/notebooks/Wire-Free-DAQ.ipynb index ff96fb3d..11c06957 100644 --- a/notebooks/Wire-Free-DAQ.ipynb +++ b/notebooks/Wire-Free-DAQ.ipynb @@ -156,10 +156,10 @@ "ename": "ValueError", "evalue": "read length must be non-negative or -1", "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mValueError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[0;32m 7\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m5000\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 8\u001b[0m \u001b[0mdataHeader\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfromstring\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m4\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0muint32\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 9\u001b[1;33m \u001b[0mdataHeader\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdataHeader\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfromstring\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdataHeader\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mBUFFER_HEADER_HEADER_LENGTH_POS\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m-\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m*\u001b[0m \u001b[1;36m4\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0muint32\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 10\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 11\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdataHeader\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;31mValueError\u001b[0m: read length must be non-negative or -1" + "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m", + "\u001B[1;31mValueError\u001B[0m Traceback (most recent call last)", + "\u001B[1;32m\u001B[0m in \u001B[0;36m\u001B[1;34m\u001B[0m\n\u001B[0;32m 7\u001B[0m \u001B[1;32mfor\u001B[0m \u001B[0mi\u001B[0m \u001B[1;32min\u001B[0m \u001B[0mrange\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;36m5000\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m 8\u001B[0m \u001B[0mdataHeader\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mnp\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mfromstring\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mf\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mread\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;36m4\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mdtype\u001B[0m\u001B[1;33m=\u001B[0m\u001B[0mnp\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0muint32\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m----> 9\u001B[1;33m \u001B[0mdataHeader\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mnp\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mappend\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mdataHeader\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mnp\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mfromstring\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mf\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mread\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mdataHeader\u001B[0m\u001B[1;33m[\u001B[0m\u001B[0mBUFFER_HEADER_HEADER_LENGTH_POS\u001B[0m\u001B[1;33m]\u001B[0m \u001B[1;33m-\u001B[0m \u001B[1;36m1\u001B[0m\u001B[1;33m)\u001B[0m \u001B[1;33m*\u001B[0m \u001B[1;36m4\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mdtype\u001B[0m\u001B[1;33m=\u001B[0m\u001B[0mnp\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0muint32\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m 10\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m 11\u001B[0m \u001B[0mprint\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mdataHeader\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n", + "\u001B[1;31mValueError\u001B[0m: read length must be non-negative or -1" ] } ], @@ -239,7 +239,7 @@ "outputs": [], "source": [ "# Delete data from SD Card\n", - "f.seek(ataStartSector * sectorSize, 0)\n", + "f.seek(dataStartSector * sectorSize, 0)\n", "\n", "zeros = []\n", "for i in range(512):\n", @@ -249,4 +249,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/notebooks/grab_frames.ipynb b/notebooks/grab_frames.ipynb index e125dde2..6c33874e 100644 --- a/notebooks/grab_frames.ipynb +++ b/notebooks/grab_frames.ipynb @@ -33,7 +33,7 @@ "warnings.filterwarnings(\"ignore\")\n", "\n", "from miniscope_io.io import SDCard\n", - "from miniscope_io.formats import WireFreeSDLayout" + "from miniscope_io.models.sdcard import SDLayout" ] }, { @@ -100,9 +100,7 @@ ] } ], - "source": [ - "pprint(WireFreeSDLayout.dict(), sort_dicts=False)" - ] + "source": "pprint(SDLayout.from_id('wirefree-sd-layout'), sort_dicts=False)" }, { "cell_type": "markdown", @@ -118,7 +116,9 @@ "In the future once I get an image we would want to make some jupyter widget to select possible drives,\n", "but for now i'll just hardcode it as a string for the sake of an example. The `SDCard` class has a\n", "`check_valid` method that looks for the `WRITE_KEY`s as in the prior notebook, so we could also\n", - "make that into a classmethod and just automatically find the right drive that way." + "make that into a classmethod and just automatically find the right drive that way.\n", + "\n", + "Rather than directly referencing the layout object, we can instead just use its `id` field" ] }, { @@ -157,7 +157,7 @@ } ], "source": [ - "sd = SDCard(drive=drive, layout = WireFreeSDLayout)\n", + "sd = SDCard(drive=drive, layout = \"wirefree-sd-layout\")\n", "\n", "pprint(sd.config.dict())" ] @@ -184984,4 +184984,4 @@ }, "nbformat": 4, "nbformat_minor": 1 -} \ No newline at end of file +} diff --git a/notebooks/plot_headers.ipynb b/notebooks/plot_headers.ipynb index ea600839..a3d98e4b 100644 --- a/notebooks/plot_headers.ipynb +++ b/notebooks/plot_headers.ipynb @@ -23,7 +23,6 @@ "warnings.filterwarnings(\"ignore\")\n", "\n", "from miniscope_io.io import SDCard\n", - "from miniscope_io.formats import WireFreeSDLayout_Battery\n", "from miniscope_io.models.data import Frames\n", "from miniscope_io.plots.headers import plot_headers, battery_voltage" ], @@ -66,7 +65,7 @@ "source": [ "# Recall that you have to use an SDCard layout that matches the data you have!\n", "# Here we are using an updated layout that includes the battery level\n", - "sd = SDCard(drive=drive, layout = WireFreeSDLayout_Battery)" + "sd = SDCard(drive=drive, layout = \"wirefree-sd-layout-battery\")" ], "metadata": { "collapsed": false, diff --git a/notebooks/save_video.ipynb b/notebooks/save_video.ipynb index 7eb08952..a04ddf66 100644 --- a/notebooks/save_video.ipynb +++ b/notebooks/save_video.ipynb @@ -21,8 +21,7 @@ "outputs": [], "source": [ "from pathlib import Path\n", - "from miniscope_io.io import SDCard\n", - "from miniscope_io.formats import WireFreeSDLayout" + "from miniscope_io.io import SDCard" ] }, { @@ -43,7 +42,7 @@ "outputs": [], "source": [ "drive = Path('..') / 'data' / 'wirefree_example.img'\n", - "sd = SDCard(drive=drive, layout = WireFreeSDLayout)" + "sd = SDCard(drive=drive, layout = \"wirefree-sd-layout\")" ], "metadata": { "collapsed": false, @@ -130,4 +129,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} \ No newline at end of file +} diff --git a/pdm.lock b/pdm.lock index 3008efc3..5fe8c706 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "all", "dev", "docs", "plot", "tests"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:bf968fd0a212fae996b67713a6df6df51a9f5751b2493341be298900fba9cb42" +content_hash = "sha256:67864847cd8a643b80e97f7722bef09d8eeb010e38ac6db0e0de325e909380b7" [[metadata.targets]] requires_python = "~=3.9" diff --git a/pyproject.toml b/pyproject.toml index d584a77d..649a60ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ dependencies = [ "rich>=13.6.0", "pyyaml>=6.0.1", "click>=8.1.7", + 'typing-extensions>=4.10.0; python_version<"3.13"' ] readme = "README.md" @@ -124,7 +125,7 @@ omit = [ ] [tool.ruff] -target-version = "py311" +target-version = "py39" include = ["miniscope_io/**/*.py", "pyproject.toml"] exclude = ["docs", "tests", "miniscope_io/vendor", "noxfile.py"] line-length = 100 diff --git a/tests/conftest.py b/tests/conftest.py index 045b48e9..2e7e55c7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,6 +6,10 @@ import pytest import yaml +from miniscope_io.models.mixins import ConfigYAMLMixin + +from .fixtures import * + DATA_DIR = Path(__file__).parent / "data" CONFIG_DIR = DATA_DIR / "config" MOCK_DIR = Path(__file__).parent / "mock" @@ -33,6 +37,21 @@ def mock_okdev(monkeypatch): monkeypatch.setattr(stream_daq, "okDev", okDevMock) +@pytest.fixture(scope="session", autouse=True) +def mock_config_source(monkeypatch_session): + """ + Add the `tests/data/config` directory to the config sources for the entire testing session + """ + current_sources = ConfigYAMLMixin.config_sources + + @classmethod + @property + def _config_sources(cls: type[ConfigYAMLMixin]) -> list[Path]: + return [CONFIG_DIR, *current_sources] + + monkeypatch_session.setattr(ConfigYAMLMixin, "config_sources", _config_sources) + + @pytest.fixture() def set_okdev_input(monkeypatch): """ diff --git a/tests/data/config/preamble_hex.yml b/tests/data/config/preamble_hex.yml index 4fce97f0..070836ef 100644 --- a/tests/data/config/preamble_hex.yml +++ b/tests/data/config/preamble_hex.yml @@ -1,3 +1,7 @@ +id: test-wireless-preamble-hex +mio_model: miniscope_io.models.stream.StreamDevConfig +mio_version: "v5.0.0" + # capture device. "OK" (Opal Kelly) or "UART" device: "OK" diff --git a/tests/data/config/stream_daq_test_200px.yml b/tests/data/config/stream_daq_test_200px.yml index 2e9fe911..3be8161a 100644 --- a/tests/data/config/stream_daq_test_200px.yml +++ b/tests/data/config/stream_daq_test_200px.yml @@ -1,3 +1,7 @@ +id: test-wireless-200px +mio_model: miniscope_io.models.stream.StreamDevConfig +mio_version: "v5.0.0" + # capture device. "OK" (Opal Kelly) or "UART" device: "OK" diff --git a/tests/data/config/wireless_example.yml b/tests/data/config/wireless_example.yml index 42496165..c200461c 100644 --- a/tests/data/config/wireless_example.yml +++ b/tests/data/config/wireless_example.yml @@ -1,3 +1,7 @@ +id: test-wireless-example +mio_model: miniscope_io.models.stream.StreamDevConfig +mio_version: "v5.0.0" + # capture device. "OK" (Opal Kelly) or "UART" device: "OK" diff --git a/tests/fixtures.py b/tests/fixtures.py index 508d2d3b..090cd8e5 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,10 +1,13 @@ from pathlib import Path +from typing import Callable, Optional import pytest +import yaml +from _pytest.monkeypatch import MonkeyPatch -from miniscope_io import SDCard -from miniscope_io.formats import WireFreeSDLayout, WireFreeSDLayout_Battery +from miniscope_io.io import SDCard from miniscope_io.models.data import Frames +from miniscope_io.models.mixins import ConfigYAMLMixin @pytest.fixture @@ -14,14 +17,14 @@ def wirefree() -> SDCard: """ sd_path = Path(__file__).parent.parent / "data" / "wirefree_example.img" - sdcard = SDCard(drive=sd_path, layout=WireFreeSDLayout) + sdcard = SDCard(drive=sd_path, layout="wirefree-sd-layout") return sdcard @pytest.fixture def wirefree_battery() -> SDCard: sd_path = Path(__file__).parent.parent / "data" / "wirefree_battery_sample.img" - sdcard = SDCard(drive=sd_path, layout=WireFreeSDLayout_Battery) + sdcard = SDCard(drive=sd_path, layout="wirefree-sd-layout-battery") return sdcard @@ -36,3 +39,62 @@ def wirefree_frames(wirefree) -> Frames: except StopIteration: break return Frames(frames=frames) + + +@pytest.fixture() +def tmp_config_source(tmp_path, monkeypatch) -> Path: + """ + Monkeypatch the config sources to include a temporary path + """ + + path = tmp_path / "configs" + path.mkdir(exist_ok=True) + current_sources = ConfigYAMLMixin.config_sources + + @classmethod + @property + def _config_sources(cls: type[ConfigYAMLMixin]) -> list[Path]: + return [path, *current_sources] + + monkeypatch.setattr(ConfigYAMLMixin, "config_sources", _config_sources) + return path + + +@pytest.fixture() +def yaml_config( + tmp_config_source, tmp_path, monkeypatch +) -> Callable[[str, dict, Optional[Path]], Path]: + out_file = tmp_config_source / "test_config.yaml" + + def _yaml_config(id: str, data: dict, path: Optional[Path] = None) -> Path: + if path is None: + path = out_file + else: + path = Path(path) + if not path.is_absolute(): + # put under tmp_path (rather than tmp_config_source) + # in case putting a file outside the config dir is intentional. + path = tmp_path / path + + if path.is_dir(): + path.mkdir(exist_ok=True, parents=True) + path = path / "test_config.yaml" + else: + path.parent.mkdir(exist_ok=True, parents=True) + + data = {"id": id, **data} + with open(path, "w") as yfile: + yaml.dump(data, yfile) + return path + + return _yaml_config + + +@pytest.fixture(scope="session") +def monkeypatch_session() -> MonkeyPatch: + """ + Monkeypatch you can use at the session scope! + """ + mpatch = MonkeyPatch() + yield mpatch + mpatch.undo() diff --git a/tests/test_formats.py b/tests/test_formats.py deleted file mode 100644 index 53b481d3..00000000 --- a/tests/test_formats.py +++ /dev/null @@ -1,49 +0,0 @@ -import pdb - -import pytest -import json -import importlib - -from miniscope_io.formats import WireFreeSDLayout - - -# More formats can be added here as needed. -@pytest.mark.parametrize('format', [WireFreeSDLayout]) -def test_to_from_json(format): - """ - A format can be exported and re-imported from JSON and remain equivalent - """ - fmt_json = format.model_dump_json() - - # convert the json to a dict - fmt_dict = json.loads(fmt_json) - - # Get the parent class - parent_class = type(format) - #parent_class_name = parent_module_str.split('.')[-1] - #parent_class = getattr(importlib.import_module(parent_module_str), parent_class_name) - - new_format = parent_class(**fmt_dict) - - assert format == new_format - - -@pytest.mark.parametrize( - ['format', 'format_json'], - [ - (WireFreeSDLayout, '{"sectors": {"header": 1022, "config": 1023, "data": 1024, "size": 512}, "write_key0": 226277911, "write_key1": 226277911, "write_key2": 226277911, "write_key3": 226277911, "word_size": 4, "header": {"gain": 4, "led": 5, "ewl": 6, "record_length": 7, "fs": 8, "delay_start": 9, "battery_cutoff": 10}, "config": {"width": 0, "height": 1, "fs": 2, "buffer_size": 3, "n_buffers_recorded": 4, "n_buffers_dropped": 5}, "buffer": {"length": 0, "linked_list": 1, "frame_num": 2, "buffer_count": 3, "frame_buffer_count": 4, "write_buffer_count": 5, "dropped_buffer_count": 6, "timestamp": 7, "data_length": 8, "write_timestamp": null, "battery_voltage": null}, "version": "0.1.1"}') - ] -) -def test_format_unchanged(format, format_json): - """ - A format is a constant and shouldn't change! - - This protects against changes in the parent classes breaking the formats, - and also breaking the formats themselves - """ - parent_class = type(format) - - format_dict = json.loads(format_json) - new_format = parent_class(**format_dict) - - assert new_format == format diff --git a/tests/test_io.py b/tests/test_io.py index a2c579c8..f4be5815 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -7,9 +7,7 @@ import numpy as np import warnings -from miniscope_io.models.data import Frame from miniscope_io.models.sdcard import SDBufferHeader -from miniscope_io.formats import WireFreeSDLayout, WireFreeSDLayout_Battery from miniscope_io.io import SDCard from miniscope_io.io import BufferedCSVWriter from miniscope_io.exceptions import EndOfRecordingException @@ -18,6 +16,7 @@ from .fixtures import wirefree, wirefree_battery + @pytest.fixture def tmp_csvfile(tmp_path): """ @@ -25,6 +24,7 @@ def tmp_csvfile(tmp_path): """ return tmp_path / "test.csv" + def test_csvwriter_initialization(tmp_csvfile): """ Test that the BufferedCSVWriter initializes correctly. @@ -34,6 +34,7 @@ def test_csvwriter_initialization(tmp_csvfile): assert writer.buffer_size == 10 assert writer.buffer == [] + def test_csvwriter_append_and_flush(tmp_csvfile): """ Test that the BufferedCSVWriter appends to the buffer and flushes it when full. @@ -41,16 +42,17 @@ def test_csvwriter_append_and_flush(tmp_csvfile): writer = BufferedCSVWriter(tmp_csvfile, buffer_size=2) writer.append([1, 2, 3]) assert len(writer.buffer) == 1 - + writer.append([4, 5, 6]) assert len(writer.buffer) == 0 assert tmp_csvfile.exists() - - with tmp_csvfile.open('r', newline='') as f: + + with tmp_csvfile.open("r", newline="") as f: reader = csv.reader(f) rows = list(reader) assert len(rows) == 2 - assert rows == [['1', '2', '3'], ['4', '5', '6']] + assert rows == [["1", "2", "3"], ["4", "5", "6"]] + def test_csvwriter_flush_buffer(tmp_csvfile): """ @@ -59,15 +61,16 @@ def test_csvwriter_flush_buffer(tmp_csvfile): writer = BufferedCSVWriter(tmp_csvfile, buffer_size=2) writer.append([1, 2, 3]) writer.flush_buffer() - + assert len(writer.buffer) == 0 assert tmp_csvfile.exists() - - with tmp_csvfile.open('r', newline='') as f: + + with tmp_csvfile.open("r", newline="") as f: reader = csv.reader(f) rows = list(reader) assert len(rows) == 1 - assert rows == [['1', '2', '3']] + assert rows == [["1", "2", "3"]] + def test_csvwriter_close(tmp_csvfile): """ @@ -76,15 +79,16 @@ def test_csvwriter_close(tmp_csvfile): writer = BufferedCSVWriter(tmp_csvfile, buffer_size=2) writer.append([1, 2, 3]) writer.close() - + assert len(writer.buffer) == 0 assert tmp_csvfile.exists() - - with tmp_csvfile.open('r', newline='') as f: + + with tmp_csvfile.open("r", newline="") as f: reader = csv.reader(f) rows = list(reader) assert len(rows) == 1 - assert rows == [['1', '2', '3']] + assert rows == [["1", "2", "3"]] + def test_read(wirefree): """ @@ -169,7 +173,7 @@ def test_relative_path(): rel_path = abs_child.relative_to(abs_cwd) assert not rel_path.is_absolute() - sdcard = SDCard(drive=rel_path, layout=WireFreeSDLayout) + sdcard = SDCard(drive=rel_path, layout="wirefree-sd-layout") # check we can do something basic like read config assert sdcard.config is not None @@ -180,7 +184,7 @@ def test_relative_path(): # now try with an absolute path abs_path = rel_path.resolve() assert abs_path.is_absolute() - sdcard_abs = SDCard(drive=abs_path, layout=WireFreeSDLayout) + sdcard_abs = SDCard(drive=abs_path, layout="wirefree-sd-layout") assert sdcard_abs.config is not None assert sdcard_abs.drive.is_absolute() @@ -212,7 +216,7 @@ def test_to_img(wirefree_battery, n_frames, hash, tmp_path): assert out_hash == hash - sd = SDCard(out_file, WireFreeSDLayout_Battery) + sd = SDCard(out_file, "wirefree-sd-layout-battery") # we should be able to read all the frames! frames = [] diff --git a/tests/test_mixins.py b/tests/test_mixins.py new file mode 100644 index 00000000..dc4e2ecb --- /dev/null +++ b/tests/test_mixins.py @@ -0,0 +1,157 @@ +from pathlib import Path +from importlib.metadata import version + +import pytest +import yaml +from pydantic import BaseModel, ConfigDict + +from miniscope_io import CONFIG_DIR +from miniscope_io.models.mixins import yaml_peek, ConfigYAMLMixin +from tests.fixtures import tmp_config_source, yaml_config + + +class NestedModel(BaseModel): + d: int = 4 + e: str = "5" + f: float = 5.5 + + +class MyModel(ConfigYAMLMixin): + id: str = "my-config" + a: int = 0 + b: str = "1" + c: float = 2.2 + child: NestedModel = NestedModel() + + +class LoaderModel(ConfigYAMLMixin): + """Model that just allows everything, only used to test write on load""" + + model_config = ConfigDict(extra="allow") + + +@pytest.mark.parametrize( + "id,path,valid", + [ + ("default-path", None, True), + ("nested-path", Path("configs/nested/path/config.yaml"), True), + ("not-valid", Path("not_in_dir/config.yaml"), False), + ], +) +def test_config_from_id(yaml_config, id, path, valid): + """Configs can be looked up with the id field if they're within a config directory""" + instance = MyModel(id=id) + yaml_config(id, instance.model_dump(), path) + if valid: + loaded = MyModel.from_id(id) + assert loaded == instance + assert loaded.child == instance.child + assert isinstance(loaded.child, NestedModel) + else: + with pytest.raises(KeyError): + MyModel.from_id(id) + + +def test_roundtrip_to_from_yaml(tmp_config_source): + """Config models can roundtrip to and from yaml""" + yaml_file = tmp_config_source / "test_config.yaml" + + instance = MyModel() + instance.to_yaml(yaml_file) + loaded = MyModel.from_yaml(yaml_file) + assert loaded == instance + assert loaded.child == instance.child + assert isinstance(loaded.child, NestedModel) + + +@pytest.mark.parametrize( + "src", + [ + pytest.param( + """ +a: 9 +b: "10\"""", + id="missing", + ), + pytest.param( + f""" +a: 9 +id: "my-config" +mio_model: "tests.test_mixins.MyModel" +mio_version: "{version('miniscope_io')}" +b: "10\"""", + id="not-at-start", + ), + pytest.param( + f""" +mio_version: "{version('miniscope_io')}" +mio_model: "tests.test_mixins.MyModel" +id: "my-config" +a: 9 +b: "10\"""", + id="out-of-order", + ), + ], +) +def test_complete_header(tmp_config_source, src: str): + """ + Config models saved without header information will have it filled in + the source yaml they were loaded from + """ + yaml_file = tmp_config_source / "test_config.yaml" + + with open(yaml_file, "w") as yfile: + yfile.write(src) + + _ = MyModel.from_yaml(yaml_file) + + with open(yaml_file, "r") as yfile: + loaded = yaml.safe_load(yfile) + + loaded_str = yaml_file.read_text() + + assert loaded["mio_version"] == version("miniscope_io") + assert loaded["id"] == "my-config" + assert loaded["mio_model"] == MyModel._model_name() + + # the header should come at the top! + lines = loaded_str.splitlines() + for i, key in enumerate(("id", "mio_model", "mio_version")): + line_key = lines[i].split(":")[0].strip() + assert line_key == key + + +@pytest.mark.parametrize("config_file", CONFIG_DIR.rglob("*.y*ml")) +def test_builtins_unchanged(config_file): + """None of the builtin configs should be modified on load - i.e. they should all have correct headers.""" + before = config_file.read_text() + _ = LoaderModel.from_yaml(config_file) + after = config_file.read_text() + assert ( + before == after + ), f"Packaged config {config_file} was modified on load, ensure it has the correct headers." + + +@pytest.mark.parametrize( + "key,expected,root,first", + [ + ("key1", "val1", True, True), + ("key1", "val1", False, True), + ("key1", ["val1"], True, False), + ("key1", ["val1", "val2"], False, False), + ("key2", "val2", True, True), + ("key3", False, True, True), + ("key4", False, True, True), + ("key4", "val4", False, True), + ], +) +def test_peek_yaml(key, expected, root, first, yaml_config): + yaml_file = yaml_config( + "test", {"key1": "val1", "key2": "val2", "key3": {"key1": "val2", "key4": "val4"}}, None + ) + + if not expected: + with pytest.raises(KeyError): + _ = yaml_peek(key, yaml_file, root=root, first=first) + else: + assert yaml_peek(key, yaml_file, root=root, first=first) == expected diff --git a/tests/test_models/test_model_buffer.py b/tests/test_models/test_model_buffer.py index 6811a424..e244c959 100644 --- a/tests/test_models/test_model_buffer.py +++ b/tests/test_models/test_model_buffer.py @@ -10,6 +10,7 @@ def test_buffer_from_format(construct): Instantiate a BufferHeader from a sequence and a format """ format = BufferHeaderFormat( + id="buffer-header", linked_list=0, frame_num=1, buffer_count=2, @@ -25,7 +26,7 @@ def test_buffer_from_format(construct): # correct vals should work in both cases instance = BufferHeader.from_format(vals, format, construct) - assert list(instance.model_dump().values()) == vals + assert list(instance.model_dump(exclude={"id"}).values()) == vals # bad vals should only work if we're constructing if construct: diff --git a/tests/test_stream_daq.py b/tests/test_stream_daq.py index 62a57301..de7b1cc5 100644 --- a/tests/test_stream_daq.py +++ b/tests/test_stream_daq.py @@ -19,8 +19,7 @@ @pytest.fixture(params=[pytest.param(5, id="buffer-size-5"), pytest.param(10, id="buffer-size-10")]) def default_streamdaq(set_okdev_input, request) -> StreamDaq: - test_config_path = CONFIG_DIR / "stream_daq_test_200px.yml" - daqConfig = StreamDevConfig.from_yaml(test_config_path) + daqConfig = StreamDevConfig.from_id("test-wireless-200px") daqConfig.runtime.frame_buffer_queue_size = request.param daqConfig.runtime.image_buffer_queue_size = request.param daqConfig.runtime.serial_buffer_queue_size = request.param @@ -37,7 +36,7 @@ def default_streamdaq(set_okdev_input, request) -> StreamDaq: "config,data,video_hash_list,show_video", [ ( - "stream_daq_test_200px.yml", + "test-wireless-200px", "stream_daq_test_fpga_raw_input_200px.bin", [ "f878f9c55de28a9ae6128631c09953214044f5b86504d6e5b0906084c64c644c", @@ -55,8 +54,7 @@ def test_video_output( ): output_video = tmp_path / "output.avi" - test_config_path = CONFIG_DIR / config - daqConfig = StreamDevConfig.from_yaml(test_config_path) + daqConfig = StreamDevConfig.from_id(config) daqConfig.runtime.frame_buffer_queue_size = buffer_size daqConfig.runtime.image_buffer_queue_size = buffer_size daqConfig.runtime.serial_buffer_queue_size = buffer_size @@ -78,14 +76,13 @@ def test_video_output( "config,data", [ ( - "stream_daq_test_200px.yml", + "test-wireless-200px", "stream_daq_test_fpga_raw_input_200px.bin", ) ], ) def test_binary_output(config, data, set_okdev_input, tmp_path): - test_config_path = CONFIG_DIR / config - daqConfig = StreamDevConfig.from_yaml(test_config_path) + daqConfig = StreamDevConfig.from_id(config) data_file = DATA_DIR / data set_okdev_input(data_file)