diff --git a/.env.sample b/.env.sample index 706fc73f..6d6fcbd9 100644 --- a/.env.sample +++ b/.env.sample @@ -1,2 +1,2 @@ -MINISCOPE_IO_BASE_DIR="~/.config/miniscope_io" -MINISCOPE_IO_LOGS__LEVEL="INFO" \ No newline at end of file +MIO_BASE_DIR="~/.config/mio" +MIO_LOGS__LEVEL="INFO" \ No newline at end of file diff --git a/.github/workflows/docs-test.yml b/.github/workflows/docs-test.yml new file mode 100644 index 00000000..29205726 --- /dev/null +++ b/.github/workflows/docs-test.yml @@ -0,0 +1,33 @@ +name: Build and test documentation + +on: + push: + branches: + - main + pull_request: + branches: + - main + workflow_dispatch: + +jobs: + test_docs: + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v3 + + - name: Set up python + uses: actions/setup-python@v4 + with: + python-version: "3.12" + cache: "pip" + + - name: Install dependencies + run: pip install -e .[docs] pytest-md + + - name: Build docs + working-directory: docs + env: + SPHINXOPTS: "-W --keep-going" + run: make html diff --git a/README.md b/README.md index 0ede6d4d..93b91838 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ -# miniscope-io +# mio: Miniscope IO -[![PyPI - Version](https://img.shields.io/pypi/v/miniscope-io)](https://pypi.org/project/miniscope_io/) +[![PyPI - Version](https://img.shields.io/pypi/v/mio)](https://pypi.org/project/mio/) [![Documentation Status](https://readthedocs.org/projects/miniscope-io/badge/?version=latest)](https://miniscope-io.readthedocs.io/en/latest/?badge=latest) -![PyPI - Python Version](https://img.shields.io/pypi/pyversions/miniscope-io) -![PyPI - Status](https://img.shields.io/pypi/status/miniscope-io) -[![Coverage Status](https://coveralls.io/repos/github/Aharoni-Lab/miniscope-io/badge.svg?branch=main)](https://coveralls.io/github/Aharoni-Lab/miniscope-io?branch=main) +![PyPI - Python Version](https://img.shields.io/pypi/pyversions/mio) +![PyPI - Status](https://img.shields.io/pypi/status/mio) +[![Coverage Status](https://coveralls.io/repos/github/Aharoni-Lab/mio/badge.svg?branch=main)](https://coveralls.io/github/Aharoni-Lab/mio?branch=main) Generic i/o interfaces for miniscopes. @@ -21,4 +21,4 @@ restrictive licenses. Those licenses can be found in the `LICENSE` files in the respective directories containing the unmodified source material -* `miniscope_io/vendor/opalkelly` \ No newline at end of file +* `mio/vendor/opalkelly` \ No newline at end of file diff --git a/docs/api/bit_operation.md b/docs/api/bit_operation.md index 04ea606c..6c0473a5 100644 --- a/docs/api/bit_operation.md +++ b/docs/api/bit_operation.md @@ -1,7 +1,7 @@ # bit operation ```{eval-rst} -.. automodule:: miniscope_io.bit_operation +.. automodule:: mio.bit_operation :members: :private-members: ``` \ No newline at end of file diff --git a/docs/api/exceptions.md b/docs/api/exceptions.md index 1ca4504f..30f675cd 100644 --- a/docs/api/exceptions.md +++ b/docs/api/exceptions.md @@ -1,7 +1,7 @@ # exceptions ```{eval-rst} -.. automodule:: miniscope_io.exceptions +.. automodule:: mio.exceptions :members: :undoc-members: ``` \ No newline at end of file diff --git a/docs/api/formats/index.md b/docs/api/formats/index.md deleted file mode 100644 index 0f181006..00000000 --- a/docs/api/formats/index.md +++ /dev/null @@ -1,12 +0,0 @@ -# formats - -```{eval-rst} -.. automodule:: miniscope_io.formats - :members: - :undoc-members: -``` - -```{toctree} -sdcard -stream -``` \ No newline at end of file diff --git a/docs/api/formats/sdcard.md b/docs/api/formats/sdcard.md deleted file mode 100644 index 60971b8a..00000000 --- a/docs/api/formats/sdcard.md +++ /dev/null @@ -1,7 +0,0 @@ -# sdcard - -```{eval-rst} -.. automodule:: miniscope_io.formats.sdcard - :members: - :undoc-members: -``` \ No newline at end of file diff --git a/docs/api/formats/stream.md b/docs/api/formats/stream.md deleted file mode 100644 index 9cca38d0..00000000 --- a/docs/api/formats/stream.md +++ /dev/null @@ -1,7 +0,0 @@ -# stream - -```{eval-rst} -.. automodule:: miniscope_io.formats.stream - :members: - :undoc-members: -``` \ No newline at end of file diff --git a/docs/api/io.md b/docs/api/io.md index e14a3726..5b343744 100644 --- a/docs/api/io.md +++ b/docs/api/io.md @@ -1,7 +1,7 @@ # io ```{eval-rst} -.. automodule:: miniscope_io.io +.. automodule:: mio.io :members: :undoc-members: ``` \ No newline at end of file diff --git a/docs/api/logging.md b/docs/api/logging.md index b7320c99..41b216ad 100644 --- a/docs/api/logging.md +++ b/docs/api/logging.md @@ -1,7 +1,7 @@ # logging ```{eval-rst} -.. automodule:: miniscope_io.logging +.. automodule:: mio.logging :members: :undoc-members: ``` \ No newline at end of file diff --git a/docs/api/models/buffer.md b/docs/api/models/buffer.md index 9ca4aec3..3da74a88 100644 --- a/docs/api/models/buffer.md +++ b/docs/api/models/buffer.md @@ -1,7 +1,7 @@ # buffer ```{eval-rst} -.. automodule:: miniscope_io.models.buffer +.. automodule:: mio.models.buffer :members: :undoc-members: ``` \ No newline at end of file diff --git a/docs/api/models/config.md b/docs/api/models/config.md index 4a450d93..c1c43f92 100644 --- a/docs/api/models/config.md +++ b/docs/api/models/config.md @@ -1,7 +1,7 @@ # config ```{eval-rst} -.. automodule:: miniscope_io.models.config +.. automodule:: mio.models.config :members: :undoc-members: ``` \ No newline at end of file diff --git a/docs/api/models/data.md b/docs/api/models/data.md index 9c8524ab..7b445859 100644 --- a/docs/api/models/data.md +++ b/docs/api/models/data.md @@ -1,7 +1,7 @@ # data ```{eval-rst} -.. automodule:: miniscope_io.data +.. automodule:: mio.data :members: :undoc-members: ``` \ No newline at end of file diff --git a/docs/api/models/index.md b/docs/api/models/index.md index 2dd8eda1..86544193 100644 --- a/docs/api/models/index.md +++ b/docs/api/models/index.md @@ -1,6 +1,6 @@ # models -Pydantic models used throughout `miniscope_io`. +Pydantic models used throughout `mio`. These models should be kept as generic as possible, and any refinements needed for a specific acquisition class should be defined within that @@ -8,6 +8,13 @@ module, inheriting from the relevant parent class. Rule of thumb: keep what is common common, and what is unique unique. + +```{eval-rst} +.. automodule:: mio.models + :members: + :undoc-members: +``` + ```{toctree} :maxdepth: 1 diff --git a/docs/api/models/mixins.md b/docs/api/models/mixins.md index d892ea63..f7ed0f8d 100644 --- a/docs/api/models/mixins.md +++ b/docs/api/models/mixins.md @@ -1,7 +1,7 @@ # mixins ```{eval-rst} -.. automodule:: miniscope_io.models.mixins +.. automodule:: mio.models.mixins :members: :undoc-members: ``` \ No newline at end of file diff --git a/docs/api/models/models.md b/docs/api/models/models.md index 916390ba..8ff2bc41 100644 --- a/docs/api/models/models.md +++ b/docs/api/models/models.md @@ -1,7 +1,7 @@ # models ```{eval-rst} -.. automodule:: miniscope_io.models.models +.. automodule:: mio.models.models :members: :undoc-members: ``` \ No newline at end of file diff --git a/docs/api/models/sdcard.md b/docs/api/models/sdcard.md index e34f4c5e..71c7fe0e 100644 --- a/docs/api/models/sdcard.md +++ b/docs/api/models/sdcard.md @@ -1,7 +1,7 @@ # sdcard ```{eval-rst} -.. automodule:: miniscope_io.models.sdcard +.. automodule:: mio.models.sdcard :members: :undoc-members: ``` \ No newline at end of file diff --git a/docs/api/models/stream.md b/docs/api/models/stream.md index dd45f77e..8966f6d6 100644 --- a/docs/api/models/stream.md +++ b/docs/api/models/stream.md @@ -1,7 +1,7 @@ # stream ```{eval-rst} -.. automodule:: miniscope_io.models.stream +.. automodule:: mio.models.stream :members: :undoc-members: ``` \ No newline at end of file diff --git a/docs/api/plots/headers.md b/docs/api/plots/headers.md index 26c57635..d6dda7f2 100644 --- a/docs/api/plots/headers.md +++ b/docs/api/plots/headers.md @@ -1,7 +1,7 @@ # headers ```{eval-rst} -.. automodule:: miniscope_io.plots.headers +.. automodule:: mio.plots.headers :members: :undoc-members: ``` \ No newline at end of file diff --git a/docs/api/stream_daq.md b/docs/api/stream_daq.md index 3f8ba06a..20d2eff5 100644 --- a/docs/api/stream_daq.md +++ b/docs/api/stream_daq.md @@ -2,13 +2,13 @@ This module is a data acquisition module that captures video streams from Miniscopes based on the `Miniscope-SAMD-Framework` firmware. The firmware repository will be published in future updates but is currently under development and private. ## Command -After [installation](../guide/installation.md) and customizing [device configurations](stream-dev-config) and [runtime configuration](models/config.md) if necessary, run the command described in [CLI Usage](../guide/cli). +After [installation](../guide/installation.md) and customizing [device configurations](stream-dev-config) and [runtime configuration](models/config.md) if necessary, run the command described in [CLI Usage](../cli/index). One example of this command is the following: ```bash $ mio stream capture -c .path/to/device/config.yml -o output_filename.avi -m ``` -A window displaying the image transferred from the Miniscope and a graph plotting metadata (`-m` option) should pop up. Additionally, the indexes of captured frames and their statuses will be logged in the terminal. The `MINISCOPE_IO_STREAM_HEADER_PLOT_KEY` defines plotted header fields (see `.env.sample`). +A window displaying the image transferred from the Miniscope and a graph plotting metadata (`-m` option) should pop up. Additionally, the indexes of captured frames and their statuses will be logged in the terminal. The `MIO_STREAM_HEADER_PLOT_KEY` defines plotted header fields (see `.env.sample`). ## Prerequisites - **Data capture hardware:** Opal Kelly XEM7310-A75 FPGA board (connected via USB) @@ -17,10 +17,10 @@ A window displaying the image transferred from the Miniscope and a graph plottin (stream-dev-config)= ## Device configuration -A YAML file is used to configure Stream DAQ based on the device configuration. The device configuration needs to match the imaging and data capture hardware for proper operation. This file is used to set up hardware, define data formats, and set data preambles. The contents of this YAML file will be parsed into a model [miniscope_io.models.stream](../api/models/stream.md), which then configures the Stream DAQ. +A YAML file is used to configure Stream DAQ based on the device configuration. The device configuration needs to match the imaging and data capture hardware for proper operation. This file is used to set up hardware, define data formats, and set data preambles. The contents of this YAML file will be parsed into a model [mio.models.stream](../api/models/stream.md), which then configures the Stream DAQ. ### FPGA (Opal Kelly) configuration -The `bitstream` field in the device configuration yaml file specifies the image that will be uploaded to the opal kelly board. This file needs to be placed in `miniscope_io.devices`. +The `bitstream` field in the device configuration yaml file specifies the image that will be uploaded to the opal kelly board. This file needs to be placed in `mio.devices`. #### Bitstream file nomenclature @@ -33,7 +33,7 @@ Name format of the bitstream files and directory: - **ENCODING_POLARITY**: Manchester encoding convention, which corresponds to bit polarity. The current package supports IEEE convention Manchester encoding. ### Example device configuration -Below is an example configuration YAML file. More examples can be found in `miniscope_io.data.config`. +Below is an example configuration YAML file. More examples can be found in `mio.data.config`. ```yaml # capture device. "OK" (Opal Kelly) or "UART" @@ -91,7 +91,7 @@ runtime: ``` ```{eval-rst} -.. automodule:: miniscope_io.stream_daq +.. automodule:: mio.stream_daq :members: :undoc-members: ``` \ No newline at end of file diff --git a/docs/api/utils.md b/docs/api/utils.md index f3939823..e7e96b3e 100644 --- a/docs/api/utils.md +++ b/docs/api/utils.md @@ -1,7 +1,7 @@ # utils ```{eval-rst} -.. automodule:: miniscope_io.utils +.. automodule:: mio.utils :members: :undoc-members: ``` \ No newline at end of file diff --git a/docs/cli/config.md b/docs/cli/config.md new file mode 100644 index 00000000..1cd4f6fd --- /dev/null +++ b/docs/cli/config.md @@ -0,0 +1,9 @@ +# `config` + +See also: [config guide](../guide/config.md) + +```{eval-rst} +.. click:: mio.cli.config:config + :prog: mio config + :nested: full +``` \ No newline at end of file diff --git a/docs/cli/device.md b/docs/cli/device.md new file mode 100644 index 00000000..b3d0ae20 --- /dev/null +++ b/docs/cli/device.md @@ -0,0 +1,6 @@ +# `device` + +```{eval-rst} +.. click:: mio.cli.main:device + :prog: mio device +``` \ No newline at end of file diff --git a/docs/cli/index.md b/docs/cli/index.md new file mode 100644 index 00000000..481c4701 --- /dev/null +++ b/docs/cli/index.md @@ -0,0 +1,19 @@ +# CLI Usage + +```{toctree} +:maxdepth: 1 + +config +device +stream +update +``` + +Refer to the following page for details regarding ``stream_daq`` device config files. + +- [stream_daq](../api/stream_daq.md) + +```{eval-rst} +.. click:: mio.cli.main:cli + :prog: mio +``` \ No newline at end of file diff --git a/docs/cli/stream.md b/docs/cli/stream.md new file mode 100644 index 00000000..a68cf924 --- /dev/null +++ b/docs/cli/stream.md @@ -0,0 +1,7 @@ +# `stream` + +```{eval-rst} +.. click:: mio.cli.stream:stream + :prog: mio stream + :nested: full +``` \ No newline at end of file diff --git a/docs/cli/update.md b/docs/cli/update.md new file mode 100644 index 00000000..50ce569b --- /dev/null +++ b/docs/cli/update.md @@ -0,0 +1,7 @@ +# `update` + +```{eval-rst} +.. click:: mio.cli.update:update + :prog: mio update + :nested: full +``` \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index d4f34e61..9ef425d6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -17,8 +17,8 @@ project = "miniscope-io" copyright = "2023, Jonny" author = "Jonny, Takuya" -release = _version("miniscope-io") -html_title = "miniscope-io" +release = _version("mio") +html_title = "mio" # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration @@ -32,8 +32,10 @@ "sphinx.ext.intersphinx", "sphinx.ext.todo", "sphinx_click", + "sphinx_design", "sphinxcontrib.autodoc_pydantic", "sphinxcontrib.mermaid", + "sphinxcontrib.programoutput", ] templates_path = ["_templates"] @@ -44,7 +46,6 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output html_theme = "furo" -html_static_path = ["_static"] intersphinx_mapping = { "python": ("https://docs.python.org/3", None), diff --git a/docs/guide/capture/wireless.md b/docs/guide/capture/wireless.md index f61f47c6..17150872 100644 --- a/docs/guide/capture/wireless.md +++ b/docs/guide/capture/wireless.md @@ -8,8 +8,8 @@ ## Header Values and Expected Transitions See following docs for the basic structure. -- `miniscope_io.models.buffer.BufferHeaderFormat` -- `miniscope_io.models.stream.StreamBufferHeaderFormat` +- `mio.models.buffer.BufferHeaderFormat` +- `mio.models.stream.StreamBufferHeaderFormat` Device specific notes are listed below. - **`preamble`**: 32-bit preamble for detecting the beginning of each buffer. The [`preamble`](../../api/stream_daq.md) in the device config needs to match the preamble defined in firmware. diff --git a/docs/guide/cli.md b/docs/guide/cli.md deleted file mode 100644 index aa4e4b14..00000000 --- a/docs/guide/cli.md +++ /dev/null @@ -1,13 +0,0 @@ -# CLI Usage - -Refer to the following page for details regarding `stream_daq` device config files. - -- [`stream_daq`](../api/stream_daq.html) - -```{eval-rst} - -.. click:: miniscope_io.cli.main:cli - :prog: mio - :nested: full - -``` \ No newline at end of file diff --git a/docs/guide/config.md b/docs/guide/config.md new file mode 100644 index 00000000..c927b364 --- /dev/null +++ b/docs/guide/config.md @@ -0,0 +1,157 @@ +# Configuration + +```{tip} +See also the API docs in {mod}`mio.models.config` +``` + +Config in `mio` uses a combination of pydantic models and +[`pydantic-settings`](https://docs.pydantic.dev/latest/concepts/pydantic_settings/). + +Configuration takes a few forms: + +- **Global config:** control over basic operation of `mio` like logging, + location of user directories, plugins, etc. +- **Device config:** control over the operation of specific devices and miniscopes like + firmware versions, ports, capture parameters, etc. +- **Runtime/experiment config:** control over how a device behaves when it runs, like + plotting, data output, etc. + +## Global Config + +Global config uses the {class}`~mio.models.config.Config` class + +Config values can be set (in order of priority from high to low, where higher +priorities override lower priorities) + +* in the arguments passed to the class constructor (not user configurable) +* in environment variables like `export MIO_LOG_DIR=~/` +* in a `.env` file in the working directory +* in a `mio_config.yaml` file in the working directory +* in the `tool.mio.config` table in a `pyproject.toml` file in the working directory +* in a user `mio_config.yaml` file in the user directory (see [below](user-directory)) +* in the global `mio_config.yaml` file in the platform-specific data directory + (use `mio config global path` to find its location) +* the default values in the {class}`~mio.models.config.Config` model + +Parent directories are _not_ checked - `.env` files, `mio_config.yaml`, and `pyproject.toml` +files need to be in the current working directory to be discovered. + +You can see your current configuration with `mio config` + +(user-directory)= +### User Directory + +The configuration system allows project-specific configs per-directory with +`mio_config.yaml` files in the working directory, as well as global configuration +via `mio_config.yaml` in the system-specific config directory +(via [platformdirs](https://pypi.org/project/platformdirs/)). +By default, `mio` does not create new directories in the user's home directory +to be polite, but the site config directory might be inconvenient or hard to reach, +so it's possible to create a user directory in a custom location. + +`mio` discovers this directory from the `user_dir` setting from +any of the available sources, though the global `mio_config.yaml` file is the most reliable. + +To create a user directory, use the `mio config user create` command. +(ignore the `--dry-run` flag, which are just used to avoid +overwriting configs while rendering the docs ;) + +```{command-output} mio config user create ~/my_new_directory --dry-run +``` + +You can confirm that this will be where mio discovers the user directory like + +```{command-output} mio config user path +``` + +If a directory is not supplied, the default `~/.config/mio` is used: + +```{command-output} mio config user create --dry-run +``` + +### Setting Values + +```{todo} +Implement setting values from CLI. + +For now, please edit the configuration files directly. +``` + +### Keys + +#### Prefix + +Keys for environment variables (i.e. set in a shell with e.g. `export` or in a `.env` file) +are prefixed with `MIO_` to not shadow other environment variables. +Keys in `toml` or `yaml` files are not prefixed with `MIO_` . + +#### Nesting + +Keys for nested models are separated by a `__` double underscore in `.env` +files or environment variables (eg. `MIO_LOGS__LEVEL`) + +Keys in `toml` or `yaml` files do not have a dunder separator because +they can represent the nesting directly (see examples below) + +When setting values from the cli, keys for nested models are separated with a `.`. + +#### Case + +Keys are case-insensitive, i.e. these are equivalent:: + + export MIO_LOGS__LEVEL=INFO + export mio_logs__level=INFO + +### Examples + +`````{tab-set} +````{tab-item} mio_config.yaml +```{code-block} yaml +user_dir: ~/.config/mio +log_dir: ~/.config/mio/logs +logs: + level_file: INFO + level_stream: WARNING + file_n: 5 +``` +```` +````{tab-item} env vars +```{code-block} bash +export MIO_USER_DIR='~/.config/mio' +export MIO_LOG_DIR='~/config/mio/logs' +export MIO_LOGS__LEVEL_FILE='INFO' +export MIO_LOGS__LEVEL_STREAM='WARNING' +export MIO_LOGS__FILE_N=5 +``` +```` +````{tab-item} .env file +```{code-block} python +MIO_USER_DIR='~/.config/mio' +MIO_LOG_DIR='~/config/mio/logs' +MIO_LOG__LEVEL_FILE='INFO' +MIO_LOG__LEVEL_STREAM='WARNING' +MIO_LOG__FILE_N=5 +``` +```` +````{tab-item} pyproject.toml +```{code-block} toml +[tool.mio.config] +user_dir = "~/.config/mio" + +[tool.linkml.config.log] +dir = "~/config/mio/logs" +level_file = "INFO" +level_stream = "WARNING" +file_n = 5 +``` +```` +````{tab-item} cli +TODO +```` +````` + +## Device Configs + +```{todo} +Document device configuration +``` \ No newline at end of file diff --git a/docs/guide/installation.md b/docs/guide/installation.md index 80f98a80..bc50b146 100644 --- a/docs/guide/installation.md +++ b/docs/guide/installation.md @@ -3,20 +3,20 @@ From PyPI: ```bash -pip install miniscope_io +pip install mio ``` From git repository, using pip: ```bash -git clone https://github.com/Aharoni-Lab/miniscope-io -cd miniscope-io +git clone https://github.com/Aharoni-Lab/mio +cd mio pip install . ``` Or pdm: ```bash -git clone https://github.com/Aharoni-Lab/miniscope-io -cd miniscope-io +git clone https://github.com/Aharoni-Lab/mio +cd mio pdm install ``` @@ -25,7 +25,7 @@ pdm install ### OpalKelly -`miniscope_io.vendor.opalkelly` - used for FPGA I/O +`mio.vendor.opalkelly` - used for FPGA I/O #### Linux @@ -46,5 +46,5 @@ No special installation should be required. #### Windows -Currently windows is not implemented - see `miniscope_io/vencor/opalkelly/README.md` for +Currently windows is not implemented - see `mio/vencor/opalkelly/README.md` for what was done to implement Linux and Mac to see what might need to be done here, pull requests welcome :) diff --git a/docs/guide/update/wireless.md b/docs/guide/update/wireless.md index b1077b03..f4c3f36e 100644 --- a/docs/guide/update/wireless.md +++ b/docs/guide/update/wireless.md @@ -2,6 +2,6 @@ **Under Construction:** This section will be populated when these devices are released. -This page will document equipments needed to use the `miniscope_io.device_update` module (or `mio update` interface.) +This page will document equipments needed to use the `mio.device_update` module (or `mio update` interface.) ## prerequisite - A custom FTDI chip based IR transmitter (details will be released soon) \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 3d42baf5..2a407ae6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,4 +1,4 @@ -# miniscope-io +# mio Generic I/O interfaces for miniscopes :) @@ -7,9 +7,10 @@ Generic I/O interfaces for miniscopes :) :maxdepth: 2 guide/installation -guide/cli +guide/config guide/capture/index guide/update/index +cli/index ``` ```{toctree} @@ -24,7 +25,6 @@ reference/object_model :caption: API: api/devices/index -api/formats/index api/io api/logging api/models/index diff --git a/docs/meta/changelog.md b/docs/meta/changelog.md index ec1c52d7..026cfa9a 100644 --- a/docs/meta/changelog.md +++ b/docs/meta/changelog.md @@ -1,5 +1,56 @@ # Changelog +## 0.6 - Becoming `mio` + +### 0.6.0 - 24-12-10 + +#### Breaking Changes + +- `miniscope-io` is now known as `mio`! Big thanks to [`@heuer`](https://github.com/Aharoni-Lab/mio/issues/77) + for graciously giving us the name. This gives us a nice, short name that is uniform + across pypi, the repository, and the cli. +- The {meth}`mio.models.config.LogConfig.level_file` and {meth}`mio.models.config.LogConfig.level_stdout` + fields are no longer automatically populated from the `level` field. + This was because of the way the multi-source config system propagates values between + sources with different priorities. Now downstream consumers should check if these values + are `None` and use the `level` field if so. + +#### Config + +Two big changes to config: + +- [`#72`](https://github.com/Aharoni-Lab/mio/pull/72) - `@sneakers-the-rat` - Global config, user config + from multiple sources: see the [config](../guide/config.md) documentation for more +- [`#76`](https://github.com/Aharoni-Lab/mio/pull/76) - `@sneakers-the-rat` - Convert `formats` to `yaml`. + We finally got rid of the godforsaken self-inflicted wound of having instantiated models + serve as config, and instead are using `yaml` everywhere for static config. This includes + every yaml-able config having a header that indicates which model the config corresponds to, + a (locally) unique id, which can be used anywhere a path can be, and a version stamp in anticipation + of being able to handle model migrations. + +#### CI + +- [`#75`](https://github.com/Aharoni-Lab/mio/pull/75) - `@sneakers-the-rat` - Test docs builds on PRs + to avoid broken links and references + +## 0.5 + +### 0.5.0 - 24-11-11 +Enhancements and bugfixes to `StreamDaq`; adding `device_update` module; CI/CD updates. + +#### Features / bugfixes +- **Over-the-air device config:** modules and commands for updating and rebooting; *e.g.,* `mio update --key LED --value 10`, `mio device --reboot`. +- **Continuous run:** updated error handling to continuously capture even when the data stream is interrupted. +- **UNIX timestamp:** added UNIX timestamp to metadata file export. +- **More Opal Kelly bitfiles:** added FPGA configuration images and organized them based on Manchester encoding conventions, frequency, etc. +#### CI/CD +- Switched to `pdm` from `poetry`; now `pdm install --with all` for contributing. +- Added workflow for readthedocs preview link in PRs. +- Added snake_case enforcement (Lint). + +Related PRs: [#45](https://github.com/Aharoni-Lab/mio/pull/45), [#48](https://github.com/Aharoni-Lab/mio/pull/48), [#49](https://github.com/Aharoni-Lab/mio/pull/49), [#50](https://github.com/Aharoni-Lab/miniscope-io/pull/50), [#53](https://github.com/Aharoni-Lab/miniscope-io/pull/53), +Contributors: [@t-sasatani](https://github.com/t-sasatani), [@sneakers-the-rat](https://github.com/sneakers-the-rat), [@MarcelMB](https://github.com/MarcelMB), [@phildong](https://github.com/phildong) + ## 0.4 ### 0.4.1 - 24-09-01 @@ -76,35 +127,35 @@ StreamDaq enhancements and testing Testing: -- [@t-sasatani](https://github.com/t-sasatani) - add end-to-end test for {class}`~miniscope_io.stream_daq.streamDaq` -- Add a mock class for {class}`~miniscope_io.devices.opalkelly.okDev` +- [@t-sasatani](https://github.com/t-sasatani) - add end-to-end test for {class}`~mio.stream_daq.streamDaq` +- Add a mock class for {class}`~mio.devices.opalkelly.okDev` - replace `tmpdir` fixture and `tempfile` module with `tmp_path` New: - [@t-sasatani](https://github.com/t-sasatani) - allow use of okDev on Windows -- {meth}`~miniscope_io.stream_daq.StreamDaq.capture` can export video :) +- {meth}`~mio.stream_daq.StreamDaq.capture` can export video :) - More specific exceptions: - - {class}`~miniscope_io.exceptions.StreamError` - - {class}`~miniscope_io.exceptions.StreamReadError` - - {class}`~miniscope_io.exceptions.DeviceError` - - {class}`~miniscope_io.exceptions.DeviceOpenError` - - {class}`~miniscope_io.exceptions.DeviceConfigurationError` -- {func}`~miniscope_io.utils.hash_video` - hash decoded video frames, rather than encoded video file + - {class}`~mio.exceptions.StreamError` + - {class}`~mio.exceptions.StreamReadError` + - {class}`~mio.exceptions.DeviceError` + - {class}`~mio.exceptions.DeviceOpenError` + - {class}`~mio.exceptions.DeviceConfigurationError` +- {func}`~mio.utils.hash_video` - hash decoded video frames, rather than encoded video file Fixed: -- Removed `print` statements in {class}`~miniscope_io.devices.opalkelly.okDev` -- {meth}`~miniscope_io.stream_daq.StreamDaq.capture` +- Removed `print` statements in {class}`~mio.devices.opalkelly.okDev` +- {meth}`~mio.stream_daq.StreamDaq.capture` - Don't require `config` - - Replace logging with {func}`~miniscope_io.logging.init_logger` - - Use of {attr}`~miniscope_io.stream_daq.StreamDaq.terminate` to control inner loops + - Replace logging with {func}`~mio.logging.init_logger` + - Use of {attr}`~mio.stream_daq.StreamDaq.terminate` to control inner loops Models: -- added `fs` and `show_video` to {class}`~miniscope_io.models.stream.StreamDaqConfig` +- added `fs` and `show_video` to {class}`~mio.models.stream.StreamDaqConfig` CI: @@ -121,7 +172,7 @@ CI: New features: -- **Support for Various Image Formats**: `streamDaq` now supports multiple image formats, including different image sizes, frame rates (FPS), and bit-depths. These configurations can be provided via a YAML file. Examples of these configurations can be found in `miniscope_io.data.config`. +- **Support for Various Image Formats**: `streamDaq` now supports multiple image formats, including different image sizes, frame rates (FPS), and bit-depths. These configurations can be provided via a YAML file. Examples of these configurations can be found in `mio.data.config`. - **Pydantic Model for Configuration**: Added a Pydantic model to validate the configuration of `streamDaq`. - **Bitstream Loader**: Added a bitstream loader to automatically configure the Opal Kelly FPGA when running `streamDaq`. - **Updated Command Line Script**: The command line script for running `streamDaq` has been updated. Use `streamDaq -c path/to/config/yaml/file.yml` to run the process with your YAML configuration file. @@ -148,12 +199,12 @@ Bugfixes: - Handle absolute paths correctly on windows, which can't deal with {meth}`pathlib.Path.resolve()`, apparently New features: -- Added {meth}`~miniscope_io.io.SDCard.to_video` to export videos - - Added notebook demonstrating {meth}`~miniscope_io.io.SDCard.to_video` -- Added {mod}`miniscope_io.utils` module with {func}`~.utils.hash_file` function for hashing files (used in testing) +- Added {meth}`~mio.io.SDCard.to_video` to export videos + - Added notebook demonstrating {meth}`~mio.io.SDCard.to_video` +- Added {mod}`mio.utils` module with {func}`~.utils.hash_file` function for hashing files (used in testing) Code structure: -- (Minor) moved {meth}`~miniscope_io.io.SDCard.skip` to general methods block (no change) +- (Minor) moved {meth}`~mio.io.SDCard.skip` to general methods block (no change) Tests: - Run tests on macos and windows @@ -180,10 +231,10 @@ Reverted: #### Additions -- Added {class}`~miniscope_io.exceptions.EndOfRecordingException` when attempting to read past last frame -- Added {attr}`~miniscope_io.io.SDCard.frame_count` property inferred from the number of buffers and buffers per frame +- Added {class}`~mio.exceptions.EndOfRecordingException` when attempting to read past last frame +- Added {attr}`~mio.io.SDCard.frame_count` property inferred from the number of buffers and buffers per frame - Return `self` when entering {class}`~.SDCard` context -- Optionally return {class}`~miniscope_io.sd.DataHeader`s from frame when reading +- Optionally return {class}`~mio.sd.DataHeader`s from frame when reading #### Bugfixes diff --git a/docs/meta/contributing.md b/docs/meta/contributing.md index df724d13..52046a44 100644 --- a/docs/meta/contributing.md +++ b/docs/meta/contributing.md @@ -4,7 +4,7 @@ Standard flow: - Fork the repository - Create a new branch from `main` - ~ do work ~ -- Open pull request against `miniscope-io:main` +- Open pull request against `mio:main` - Code review and discussion happens - Merge contribution @@ -37,7 +37,7 @@ pip install '.[all]' ### Linting -`miniscope-io` uses `black` for code formatting and `ruff` for linting. +`mio` uses `black` for code formatting and `ruff` for linting. We recommend you configure your IDE to do both automatically. There are a few ways you can run linting manually: @@ -46,7 +46,7 @@ First, just by running the raw commands: ```shell ruff check --fix -black miniscope_io +black mio ``` Or using pdm scripts diff --git a/miniscope_io/cli/__main__.py b/miniscope_io/cli/__main__.py deleted file mode 100644 index 07cbb180..00000000 --- a/miniscope_io/cli/__main__.py +++ /dev/null @@ -1,8 +0,0 @@ -""" -Directly call miniscope-io cli from python module rather than entrypoint script `mio` -""" - -from miniscope_io.cli.main import cli - -if __name__ == "__main__": - cli() diff --git a/miniscope_io/formats/__init__.py b/miniscope_io/formats/__init__.py deleted file mode 100644 index 5ab7fe30..00000000 --- a/miniscope_io/formats/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -""" -Instantiations of :class:`~.miniscope_io.models.MiniscopeConfig` models -that describe fixed per-device configurations for the generic config -models in :mod:`~.miniscope_io.models.stream` et al. -""" - -from miniscope_io.formats.sdcard import WireFreeSDLayout, WireFreeSDLayout_Battery - -__all__ = ["WireFreeSDLayout", "WireFreeSDLayout_Battery"] diff --git a/miniscope_io/formats/sdcard.py b/miniscope_io/formats/sdcard.py deleted file mode 100644 index 04dffc14..00000000 --- a/miniscope_io/formats/sdcard.py +++ /dev/null @@ -1,80 +0,0 @@ -""" -SD Card data layout formats for different miniscopes! -""" - -from miniscope_io.models.sdcard import ( - ConfigPositions, - SDBufferHeaderFormat, - SDHeaderPositions, - SDLayout, - SectorConfig, -) - -WireFreeSDLayout = SDLayout( - version="0.1.1", - sectors=SectorConfig(header=1022, config=1023, data=1024, size=512), - write_key0=0x0D7CBA17, - write_key1=0x0D7CBA17, - write_key2=0x0D7CBA17, - write_key3=0x0D7CBA17, - header=SDHeaderPositions( - gain=4, led=5, ewl=6, record_length=7, fs=8, delay_start=9, battery_cutoff=10 - ), - config=ConfigPositions( - width=0, - height=1, - fs=2, - buffer_size=3, - n_buffers_recorded=4, - n_buffers_dropped=5, - ), - buffer=SDBufferHeaderFormat( - length=0, - linked_list=1, - frame_num=2, - buffer_count=3, - frame_buffer_count=4, - write_buffer_count=5, - dropped_buffer_count=6, - timestamp=7, - data_length=8, - ), -) - -WireFreeSDLayout_Battery = SDLayout(**WireFreeSDLayout.model_dump()) -""" -Making another format for now, but added version field so that we could -replace making more top-level classes with a FormatCollection that can store -sets of formats for the same device with multiple versions. -""" -WireFreeSDLayout_Battery.buffer.write_timestamp = 9 -WireFreeSDLayout_Battery.buffer.battery_voltage = 10 - - -WireFreeSDLayout_Old = SDLayout( - sectors=SectorConfig(header=1023, config=1024, data=1025, size=512), - write_key0=0x0D7CBA17, - write_key1=0x0D7CBA17, - write_key2=0x0D7CBA17, - write_key3=0x0D7CBA17, - header=SDHeaderPositions(gain=4, led=5, ewl=6, record_length=7, fs=8), - config=ConfigPositions( - width=0, - height=1, - fs=2, - buffer_size=3, - n_buffers_recorded=4, - n_buffers_dropped=5, - ), - buffer=SDBufferHeaderFormat( - length=0, - linked_list=1, - frame_num=2, - buffer_count=3, - frame_buffer_count=4, - write_buffer_count=5, - dropped_buffer_count=6, - timestamp=7, - data_length=8, - ), -) diff --git a/miniscope_io/formats/stream.py b/miniscope_io/formats/stream.py deleted file mode 100644 index 4786491e..00000000 --- a/miniscope_io/formats/stream.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -Formats for use with :mod:`miniscope_io.stream_daq` -We plan to re-define this soon so documentation will come after that. -""" - -from miniscope_io.models.stream import StreamBufferHeaderFormat - -StreamBufferHeader = StreamBufferHeaderFormat( - linked_list=0, - frame_num=1, - buffer_count=2, - frame_buffer_count=3, - write_buffer_count=4, - dropped_buffer_count=5, - timestamp=6, - pixel_count=7, - write_timestamp=8, - battery_voltage_raw=9, - input_voltage_raw=10, -) diff --git a/miniscope_io/models/config.py b/miniscope_io/models/config.py deleted file mode 100644 index 51f096e8..00000000 --- a/miniscope_io/models/config.py +++ /dev/null @@ -1,118 +0,0 @@ -""" -Module-global configuration models -""" - -from pathlib import Path -from typing import Literal, Optional - -from pydantic import Field, field_validator, model_validator -from pydantic_settings import BaseSettings, SettingsConfigDict - -from miniscope_io.models import MiniscopeIOModel - -_default_basedir = Path().home() / ".config" / "miniscope_io" -LOG_LEVELS = Literal["DEBUG", "INFO", "WARNING", "ERROR"] - - -class LogConfig(MiniscopeIOModel): - """ - Configuration for logging - """ - - level: LOG_LEVELS = "INFO" - """ - Severity of log messages to process. - """ - level_file: Optional[LOG_LEVELS] = None - """ - Severity for file-based logging. If unset, use ``level`` - """ - level_stdout: Optional[LOG_LEVELS] = None - """ - Severity for stream-based logging. If unset, use ``level`` - """ - file_n: int = 5 - """ - Number of log files to rotate through - """ - file_size: int = 2**22 # roughly 4MB - """ - Maximum size of log files (bytes) - """ - - @field_validator("level", "level_file", "level_stdout", mode="before") - @classmethod - def uppercase_levels(cls, value: Optional[str] = None) -> Optional[str]: - """ - Ensure log level strings are uppercased - """ - if value is not None: - value = value.upper() - return value - - @model_validator(mode="after") - def inherit_base_level(self) -> "LogConfig": - """ - If loglevels for specific output streams are unset, set from base :attr:`.level` - """ - levels = ("level_file", "level_stdout") - for level_name in levels: - if getattr(self, level_name) is None: - setattr(self, level_name, self.level) - return self - - -class Config(BaseSettings): - """ - Runtime configuration for miniscope-io. - - See https://docs.pydantic.dev/latest/concepts/pydantic_settings/ - - Set values either in an ``.env`` file or using environment variables - prefixed with ``MINISCOPE_IO_*``. Values in nested models are separated with ``__`` , - eg. ``MINISCOPE_IO_LOGS__LEVEL`` - - See ``.env.example`` in repository root - - Paths are set relative to ``base_dir`` by default, unless explicitly specified. - - - """ - - base_dir: Path = Field( - _default_basedir, - description="Base directory to store configuration and other temporary files, " - "other paths are relative to this by default", - ) - log_dir: Path = Field(Path("logs"), description="Location to store logs") - logs: LogConfig = Field(LogConfig(), description="Additional settings for logs") - - @field_validator("base_dir", mode="before") - @classmethod - def folder_exists(cls, v: Path) -> Path: - """Ensure base_dir exists, make it otherwise""" - v = Path(v) - v.mkdir(exist_ok=True, parents=True) - - assert v.exists(), f"{v} does not exist!" - return v - - @model_validator(mode="after") - def paths_relative_to_basedir(self) -> "Config": - """If relative paths are given, make them absolute relative to ``base_dir``""" - paths = ("log_dir",) - for path_name in paths: - path = getattr(self, path_name) # type: Path - if not path.is_absolute(): - path = self.base_dir / path - setattr(self, path_name, path) - path.mkdir(exist_ok=True) - assert path.exists() - return self - - model_config = SettingsConfigDict( - env_prefix="miniscope_io_", - env_file=".env", - env_nested_delimiter="__", - extra="ignore", - ) diff --git a/miniscope_io/models/mixins.py b/miniscope_io/models/mixins.py deleted file mode 100644 index 18723ce3..00000000 --- a/miniscope_io/models/mixins.py +++ /dev/null @@ -1,25 +0,0 @@ -""" -Mixin classes that are to be used alongside specific models -to use composition for functionality and inheritance for semantics. -""" - -from pathlib import Path -from typing import Type, TypeVar, Union - -import yaml - -T = TypeVar("T") - - -class YAMLMixin: - """ - Mixin class that provides :meth:`.from_yaml` and :meth:`.to_yaml` - classmethods - """ - - @classmethod - def from_yaml(cls: Type[T], file_path: Union[str, Path]) -> T: - """Instantiate this class by passing the contents of a yaml file as kwargs""" - with open(file_path) as file: - config_data = yaml.safe_load(file) - return cls(**config_data) diff --git a/miniscope_io/sources/__init__.py b/miniscope_io/sources/__init__.py deleted file mode 100644 index 0b9acc87..00000000 --- a/miniscope_io/sources/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -""" -Control interfaces for external hardware sources -""" - -from miniscope_io.sources.opalkelly import okDev - -__all__ = ["okDev"] diff --git a/miniscope_io/types.py b/miniscope_io/types.py deleted file mode 100644 index 98df4f2f..00000000 --- a/miniscope_io/types.py +++ /dev/null @@ -1,34 +0,0 @@ -""" -Type and type annotations -""" - -from typing import NamedTuple, Tuple, Union - -Range = Union[Tuple[int, int], Tuple[float, float]] - - -class BBox(NamedTuple): - """ - Bounding Box - - (for specificying a rectangular ROI within an image frame) - """ - - x: int - """Leftmost x coordinate""" - y: int - """Topmost y coordinate""" - width: int - height: int - - -class Resolution(NamedTuple): - """ - Pixel resolution of a frame or camera. - - (i.e. the number of pixels a frame is wide and tall, - not e.g. the spatial extent of an individual pixel) - """ - - width: int - height: int diff --git a/miniscope_io/vendor/opalkelly/__init__.py b/miniscope_io/vendor/opalkelly/__init__.py deleted file mode 100644 index e09a1ebb..00000000 --- a/miniscope_io/vendor/opalkelly/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -import sys -from pathlib import Path -import os - - -def patch_env_path(name:str, value:str): - val = os.environ.get(name, None) - if val is None: - val = value - else: - val = ':'.join([val.rstrip(':'), value]) - os.environ[name] = val - -base_path = Path(__file__).parent.resolve() - -if sys.platform == 'darwin': - from miniscope_io.vendor.opalkelly.mac.ok import * - -elif sys.platform.startswith('linux'): - # Linux - from miniscope_io.vendor.opalkelly.linux.ok import * - -elif sys.platform.startswith('win'): - from miniscope_io.vendor.opalkelly.win.ok import * -else: - raise ImportError('Dont know what operating system you are on, cant use OpalKelly') diff --git a/miniscope_io/__init__.py b/mio/__init__.py similarity index 56% rename from miniscope_io/__init__.py rename to mio/__init__.py index 4fecb274..63698628 100644 --- a/miniscope_io/__init__.py +++ b/mio/__init__.py @@ -2,11 +2,11 @@ I/O SDK for UCLA Miniscopes """ +from importlib import metadata from pathlib import Path -from miniscope_io.io import SDCard -from miniscope_io.logging import init_logger -from miniscope_io.models.config import Config +from mio.logging import init_logger +from mio.models.config import Config BASE_DIR = Path(__file__).parent.resolve() DATA_DIR = BASE_DIR / "data" @@ -18,6 +18,10 @@ "DATA_DIR", "CONFIG_DIR", "Config", - "SDCard", "init_logger", ] + +try: + __version__ = metadata.version("mio") +except metadata.PackageNotFoundError: # pragma: nocover + __version__ = None diff --git a/miniscope_io/bit_operation.py b/mio/bit_operation.py similarity index 100% rename from miniscope_io/bit_operation.py rename to mio/bit_operation.py diff --git a/miniscope_io/cli/__init__.py b/mio/cli/__init__.py similarity index 100% rename from miniscope_io/cli/__init__.py rename to mio/cli/__init__.py diff --git a/mio/cli/__main__.py b/mio/cli/__main__.py new file mode 100644 index 00000000..5fd84723 --- /dev/null +++ b/mio/cli/__main__.py @@ -0,0 +1,8 @@ +""" +Directly call mio cli from python module rather than entrypoint script `mio` +""" + +from mio.cli.main import cli + +if __name__ == "__main__": + cli() diff --git a/mio/cli/common.py b/mio/cli/common.py new file mode 100644 index 00000000..0d09f36e --- /dev/null +++ b/mio/cli/common.py @@ -0,0 +1,31 @@ +""" +Shared CLI utils +""" + +from os import PathLike +from pathlib import Path +from typing import Optional + +from click import Context, Parameter, ParamType + + +class ConfigIDOrPath(ParamType): + """ + A custom click type to accept either a config `id` or a path + as input, resolving relative paths first against + the current working directory and second against the user config directory. + """ + + name = "config-id-or-path" + + def convert( + self, value: str | PathLike[str], param: Optional[Parameter], ctx: Optional[Context] + ) -> str | Path: + """ + If something looks like a yaml file, return as a path, otherwise return unchanged. + + Don't do validation here, the Config model will handle that on instantiation. + """ + if value.endswith(".yaml") or value.endswith(".yml"): + value = Path(value) + return value diff --git a/mio/cli/config.py b/mio/cli/config.py new file mode 100644 index 00000000..d8653d30 --- /dev/null +++ b/mio/cli/config.py @@ -0,0 +1,148 @@ +""" +CLI commands for configuration +""" + +from pathlib import Path + +import click +import yaml + +from mio.models import config as _config +from mio.models.config import set_user_dir + + +@click.group(invoke_without_command=True) +@click.pass_context +def config(ctx: click.Context) -> None: + """ + Command group for config + + When run without arguments, displays current config from all sources + """ + if ctx.invoked_subcommand is None: + config_str = _config.Config().to_yaml() + click.echo(f"mio configuration:\n-----\n{config_str}") + + +@config.group("global", invoke_without_command=True) +@click.pass_context +def global_(ctx: click.Context) -> None: + """ + Command group for global configuration directory + + When run without arguments, displays contents of current global config + """ + if ctx.invoked_subcommand is None: + + with open(_config._global_config_path) as f: + config_str = f.read() + + click.echo(f"Global configuration: {str(_config._global_config_path)}\n-----\n{config_str}") + + +@global_.command("path") +def global_path() -> None: + """Location of the global mio config""" + click.echo(str(_config._global_config_path)) + + +@config.group(invoke_without_command=True) +@click.pass_context +def user(ctx: click.Context) -> None: + """ + Command group for the user config directory + + When invoked without arguments, displays the contents of the current user directory + """ + if ctx.invoked_subcommand is None: + config = _config.Config() + config_file = list(config.user_dir.glob("mio_config.*")) + if len(config_file) == 0: + click.echo( + f"User directory specified as {str(config.user_dir)} " + "but no mio_config.yaml file found" + ) + return + else: + config_file = config_file[0] + + with open(config_file) as f: + config_str = f.read() + + click.echo(f"User configuration: {str(config_file)}\n-----\n{config_str}") + + +@user.command("create") +@click.argument("user_dir", type=click.Path(), required=False) +@click.option( + "--force/--no-force", + default=False, + help="Overwrite existing config file if it exists", +) +@click.option( + "--clean/--dirty", + default=False, + help="Create a fresh mio_config.yaml file containing only the user_dir. " + "Otherwise, by default (--dirty), any other settings from .env, pyproject.toml, etc." + "are included in the created user config file.", +) +@click.option( + "--dry-run/--no-dry-run", + default=False, + help="Show the config that would be written and where it would go without doing anything", +) +def create( + user_dir: Path = None, force: bool = False, clean: bool = False, dry_run: bool = False +) -> None: + """ + Create a user directory, + setting it as the default in the global config + + Args: + user_dir (Path): Path to the directory to create + force (bool): Overwrite existing config file if it exists + """ + if user_dir is None: + user_dir = _config._default_userdir + + try: + user_dir = Path(user_dir).expanduser().resolve() + except RuntimeError: + user_dir = Path(user_dir).resolve() + + if user_dir.is_file and user_dir.suffix in (".yaml", ".yml"): + config_file = user_dir + user_dir = user_dir.parent + else: + config_file = user_dir / "mio_config.yaml" + + if config_file.exists() and not force and not dry_run: + click.echo(f"Config file already exists at {str(config_file)}, use --force to overwrite") + return + + if clean: + config = {"user_dir": str(user_dir)} + + if not dry_run: + with open(config_file, "w") as f: + yaml.safe_dump(config, f) + + config_str = yaml.safe_dump(config) + else: + config = _config.Config(user_dir=user_dir) + config_str = config.to_yaml() if dry_run else config.to_yaml(config_file) + + # update global config pointer + if not dry_run: + set_user_dir(user_dir) + + prefix = "DRY RUN - No files changed\n-----\nWould have created" if dry_run else "Created" + + click.echo(f"{prefix} user config at {str(config_file)}:\n-----\n{config_str}") + + +@user.command("path") +def user_path() -> None: + """Location of the current user config""" + path = list(_config.Config().user_dir.glob("mio_config.*"))[0] + click.echo(str(path)) diff --git a/miniscope_io/cli/main.py b/mio/cli/main.py similarity index 59% rename from miniscope_io/cli/main.py rename to mio/cli/main.py index 68d0fabb..f9c85413 100644 --- a/miniscope_io/cli/main.py +++ b/mio/cli/main.py @@ -4,12 +4,13 @@ import click -from miniscope_io.cli.stream import stream -from miniscope_io.cli.update import device, update +from mio.cli.config import config +from mio.cli.stream import stream +from mio.cli.update import device, update @click.group() -@click.version_option(package_name="miniscope_io") +@click.version_option(package_name="mio") @click.pass_context def cli(ctx: click.Context) -> None: """ @@ -21,3 +22,4 @@ def cli(ctx: click.Context) -> None: cli.add_command(stream) cli.add_command(update) cli.add_command(device) +cli.add_command(config) diff --git a/miniscope_io/cli/stream.py b/mio/cli/stream.py similarity index 89% rename from miniscope_io/cli/stream.py rename to mio/cli/stream.py index 661db535..a0f07a6c 100644 --- a/miniscope_io/cli/stream.py +++ b/mio/cli/stream.py @@ -8,7 +8,8 @@ import click -from miniscope_io.stream_daq import StreamDaq +from mio.cli.common import ConfigIDOrPath +from mio.stream_daq import StreamDaq @click.group() @@ -24,8 +25,13 @@ def _common_options(fn: Callable) -> Callable: "-c", "--device_config", required=True, - help="Path to device config YAML file for streamDaq (see models.stream.StreamDevConfig)", - type=click.Path(exists=True), + help=( + "Either a config `id` or a path to device config YAML file for streamDaq " + "(see models.stream.StreamDevConfig). If path is relative, treated as " + "relative to the current directory, and then if no matching file is found, " + "relative to the user `config_dir` (see `mio config --help`)." + ), + type=ConfigIDOrPath(), )(fn) return fn @@ -56,6 +62,7 @@ def _capture_options(fn: Callable) -> Callable: help="Display metadata in real time. \n" "**WARNING:** This is still an **EXPERIMENTAL** feature and is **UNSTABLE**.", )(fn) + return fn diff --git a/miniscope_io/cli/update.py b/mio/cli/update.py similarity index 82% rename from miniscope_io/cli/update.py rename to mio/cli/update.py index b809a446..5ea980c4 100644 --- a/miniscope_io/cli/update.py +++ b/mio/cli/update.py @@ -5,8 +5,8 @@ import click import yaml -from miniscope_io.device_update import device_update -from miniscope_io.models.devupdate import DeviceCommand +from mio.device_update import device_update +from mio.models.devupdate import DeviceCommand @click.command() @@ -28,8 +28,8 @@ "-k", "--key", required=False, - type=click.Choice(["LED", "GAIN", "ROI_X", "ROI_Y"]), - help="key to update. Cannot be used with --restart.", + type=click.Choice(["LED", "GAIN", "ROI_X", "ROI_Y", "SUBSAMPLE"]), + help="key to update.", ) @click.option( "-v", @@ -52,16 +52,13 @@ def update(port: str, key: str, value: int, device_id: int, batch: str) -> None: """ Update device configuration. """ - # Check mutual exclusivity - if (key and not value) or (value and not key): + if (key and value is None) or (value and not key): raise click.UsageError("Both --key and --value are required if one is specified.") if batch and (key or value): - raise click.UsageError( - "Options --key/--value and --restart" " and --batch are mutually exclusive." - ) - if key and value: + raise click.UsageError("Options --key/--value and --batch are mutually exclusive.") + if key and value is not None: device_update(port=port, key=key, value=value, device_id=device_id) elif batch: with open(batch) as f: @@ -91,7 +88,7 @@ def update(port: str, key: str, value: int, device_id: int, batch: str) -> None: "--reboot", is_flag=True, type=bool, - help="Restart the device. Cannot be used with --key or --value.", + help="Restart the device.", ) def device(port: str, device_id: int, reboot: bool) -> None: """ diff --git a/miniscope_io/data/bitfile/XEM7310-A75/USBInterface-10mhz-J2_2-3v3-IEEE.bit b/mio/data/bitfile/USBInterface-10mhz-J2_2-3v3-IEEE.bit similarity index 100% rename from miniscope_io/data/bitfile/XEM7310-A75/USBInterface-10mhz-J2_2-3v3-IEEE.bit rename to mio/data/bitfile/USBInterface-10mhz-J2_2-3v3-IEEE.bit diff --git a/miniscope_io/data/bitfile/XEM7310-A75/USBInterface-12_5mhz-J2_2-3v3-IEEE.bit b/mio/data/bitfile/USBInterface-12_5mhz-J2_2-3v3-IEEE.bit similarity index 100% rename from miniscope_io/data/bitfile/XEM7310-A75/USBInterface-12_5mhz-J2_2-3v3-IEEE.bit rename to mio/data/bitfile/USBInterface-12_5mhz-J2_2-3v3-IEEE.bit diff --git a/miniscope_io/data/bitfile/XEM7310-A75/USBInterface-14_3mhz-J2_2-3v3-IEEE.bit b/mio/data/bitfile/USBInterface-14_3mhz-J2_2-3v3-IEEE.bit similarity index 100% rename from miniscope_io/data/bitfile/XEM7310-A75/USBInterface-14_3mhz-J2_2-3v3-IEEE.bit rename to mio/data/bitfile/USBInterface-14_3mhz-J2_2-3v3-IEEE.bit diff --git a/miniscope_io/data/bitfile/XEM7310-A75/USBInterface-3_03mhz-J2_2-3v3-IEEE.bit b/mio/data/bitfile/USBInterface-3_03mhz-J2_2-3v3-IEEE.bit similarity index 100% rename from miniscope_io/data/bitfile/XEM7310-A75/USBInterface-3_03mhz-J2_2-3v3-IEEE.bit rename to mio/data/bitfile/USBInterface-3_03mhz-J2_2-3v3-IEEE.bit diff --git a/miniscope_io/data/bitfile/XEM7310-A75/USBInterface-5mhz-J2_2-3v3-IEEE.bit b/mio/data/bitfile/USBInterface-5mhz-J2_2-3v3-IEEE.bit similarity index 100% rename from miniscope_io/data/bitfile/XEM7310-A75/USBInterface-5mhz-J2_2-3v3-IEEE.bit rename to mio/data/bitfile/USBInterface-5mhz-J2_2-3v3-IEEE.bit diff --git a/miniscope_io/data/bitfile/XEM7310-A75/USBInterface-6_67mhz-J2_2-3v3-IEEE.bit b/mio/data/bitfile/USBInterface-6_67mhz-J2_2-3v3-IEEE.bit similarity index 100% rename from miniscope_io/data/bitfile/XEM7310-A75/USBInterface-6_67mhz-J2_2-3v3-IEEE.bit rename to mio/data/bitfile/USBInterface-6_67mhz-J2_2-3v3-IEEE.bit diff --git a/miniscope_io/data/bitfile/XEM7310-A75/USBInterface-8_33mhz-J2_2-3v3-IEEE.bit b/mio/data/bitfile/USBInterface-8_33mhz-J2_2-3v3-IEEE.bit similarity index 100% rename from miniscope_io/data/bitfile/XEM7310-A75/USBInterface-8_33mhz-J2_2-3v3-IEEE.bit rename to mio/data/bitfile/USBInterface-8_33mhz-J2_2-3v3-IEEE.bit diff --git a/mio/data/config/wirefree/sd-layout-battery.yaml b/mio/data/config/wirefree/sd-layout-battery.yaml new file mode 100644 index 00000000..e52788ea --- /dev/null +++ b/mio/data/config/wirefree/sd-layout-battery.yaml @@ -0,0 +1,41 @@ +id: wirefree-sd-layout-battery +mio_model: mio.models.sdcard.SDLayout +mio_version: v5.0.0 +sectors: + header: 1022 + config: 1023 + data: 1024 + size: 512 +write_key0: 226277911 +write_key1: 226277911 +write_key2: 226277911 +write_key3: 226277911 +word_size: 4 +header: + gain: 4 + led: 5 + ewl: 6 + record_length: 7 + fs: 8 + delay_start: 9 + battery_cutoff: 10 +config: + width: 0 + height: 1 + fs: 2 + buffer_size: 3 + n_buffers_recorded: 4 + n_buffers_dropped: 5 +buffer: + linked_list: 1 + frame_num: 2 + buffer_count: 3 + frame_buffer_count: 4 + write_buffer_count: 5 + dropped_buffer_count: 6 + timestamp: 7 + write_timestamp: 9 + length: 0 + data_length: 8 + battery_voltage: 10 +version: 0.1.1 diff --git a/mio/data/config/wirefree/sd-layout.yaml b/mio/data/config/wirefree/sd-layout.yaml new file mode 100644 index 00000000..da0c08b2 --- /dev/null +++ b/mio/data/config/wirefree/sd-layout.yaml @@ -0,0 +1,40 @@ +id: wirefree-sd-layout +mio_model: mio.models.sdcard.SDLayout +mio_version: v5.0.0 +sectors: + header: 1022 + config: 1023 + data: 1024 + size: 512 +write_key0: 226277911 +write_key1: 226277911 +write_key2: 226277911 +write_key3: 226277911 +word_size: 4 +header: + gain: 4 + led: 5 + ewl: 6 + record_length: 7 + fs: 8 + delay_start: 9 + battery_cutoff: 10 +config: + width: 0 + height: 1 + fs: 2 + buffer_size: 3 + n_buffers_recorded: 4 + n_buffers_dropped: 5 +buffer: + linked_list: 1 + frame_num: 2 + buffer_count: 3 + frame_buffer_count: 4 + write_buffer_count: 5 + dropped_buffer_count: 6 + timestamp: 7 + write_timestamp: null + length: 0 + data_length: 8 + battery_voltage: null diff --git a/mio/data/config/wireless/stream-buffer-header.yaml b/mio/data/config/wireless/stream-buffer-header.yaml new file mode 100644 index 00000000..fd4610c4 --- /dev/null +++ b/mio/data/config/wireless/stream-buffer-header.yaml @@ -0,0 +1,14 @@ +id: stream-buffer-header +mio_model: mio.models.stream.StreamBufferHeaderFormat +mio_version: "v5.0.0" +linked_list: 0 +frame_num: 1 +buffer_count: 2 +frame_buffer_count: 3 +write_buffer_count: 4 +dropped_buffer_count: 5 +timestamp: 6 +write_timestamp: 8 +pixel_count: 7 +battery_voltage_raw: 9 +input_voltage_raw: 10 diff --git a/miniscope_io/data/config/WLMS_v02_200px.yml b/mio/data/config/wireless/wireless-200px.yml similarity index 93% rename from miniscope_io/data/config/WLMS_v02_200px.yml rename to mio/data/config/wireless/wireless-200px.yml index e5ef3cff..75aa4ad3 100644 --- a/miniscope_io/data/config/WLMS_v02_200px.yml +++ b/mio/data/config/wireless/wireless-200px.yml @@ -1,3 +1,7 @@ +id: wireless-200px +mio_model: mio.models.stream.StreamDevConfig +mio_version: "v5.0.0" + # capture device. "OK" (Opal Kelly) or "UART" device: "OK" diff --git a/miniscope_io/device_update.py b/mio/device_update.py similarity index 78% rename from miniscope_io/device_update.py rename to mio/device_update.py index 9680794e..5e01cc7c 100644 --- a/miniscope_io/device_update.py +++ b/mio/device_update.py @@ -8,16 +8,16 @@ import serial import serial.tools.list_ports -from miniscope_io.logging import init_logger -from miniscope_io.models.devupdate import DevUpdateCommand, UpdateCommandDefinitions +from mio.logging import init_logger +from mio.models.devupdate import DevUpdateCommand, UpdateCommandDefinitions -logger = init_logger(name="device_update", level="INFO") +logger = init_logger(name="device_update") FTDI_VENDOR_ID = 0x0403 FTDI_PRODUCT_ID = 0x6001 def device_update( - target: str, + key: str, value: int, device_id: int, port: Optional[str] = None, @@ -28,8 +28,8 @@ def device_update( Args: device_id: ID of the device. 0 will update all devices. port: Serial port to which the device is connected. - target: What to update on the device (e.g., LED, GAIN). - value: Value to which the target should be updated. + key: What to update on the device (e.g., LED, GAIN). + value: Value to which the key should be updated. Returns: None @@ -47,8 +47,8 @@ def device_update( port = ftdi_port_list[0] logger.info(f"Using port {port}") - command = DevUpdateCommand(device_id=device_id, port=port, target=target, value=value) - logger.info(f"Updating {target} to {value} on port {port}") + command = DevUpdateCommand(device_id=device_id, port=port, key=key, value=value) + logger.info(f"Updating {key} to {value} on port {port}") try: serial_port = serial.Serial(port=command.port, baudrate=2400, timeout=5, stopbits=2) @@ -63,9 +63,9 @@ def device_update( logger.debug(f"Command: {format(id_command, '08b')}; Device ID: {command.device_id}") time.sleep(0.1) - target_command = (command.target.value + UpdateCommandDefinitions.target_header) & 0xFF - serial_port.write(target_command.to_bytes(1, "big")) - logger.debug(f"Command: {format(target_command, '08b')}; Target: {command.target.name}") + key_command = (command.key.value + UpdateCommandDefinitions.key_header) & 0xFF + serial_port.write(key_command.to_bytes(1, "big")) + logger.debug(f"Command: {format(key_command, '08b')}; Key: {command.key.name}") time.sleep(0.1) value_LSB_command = ( diff --git a/miniscope_io/devices/__init__.py b/mio/devices/__init__.py similarity index 100% rename from miniscope_io/devices/__init__.py rename to mio/devices/__init__.py diff --git a/miniscope_io/devices/camera.py b/mio/devices/camera.py similarity index 100% rename from miniscope_io/devices/camera.py rename to mio/devices/camera.py diff --git a/miniscope_io/devices/device.py b/mio/devices/device.py similarity index 100% rename from miniscope_io/devices/device.py rename to mio/devices/device.py diff --git a/miniscope_io/exceptions.py b/mio/exceptions.py similarity index 100% rename from miniscope_io/exceptions.py rename to mio/exceptions.py diff --git a/miniscope_io/io.py b/mio/io.py similarity index 98% rename from miniscope_io/io.py rename to mio/io.py index 3b902e94..e6227775 100644 --- a/miniscope_io/io.py +++ b/mio/io.py @@ -12,10 +12,11 @@ import numpy as np from tqdm import tqdm -from miniscope_io.exceptions import EndOfRecordingException, ReadHeaderException -from miniscope_io.logging import init_logger -from miniscope_io.models.data import Frame -from miniscope_io.models.sdcard import SDBufferHeader, SDConfig, SDLayout +from mio.exceptions import EndOfRecordingException, ReadHeaderException +from mio.logging import init_logger +from mio.models.data import Frame +from mio.models.sdcard import SDBufferHeader, SDConfig, SDLayout +from mio.types import ConfigSource class BufferedCSVWriter: @@ -104,9 +105,11 @@ class SDCard: """ - def __init__(self, drive: Union[str, Path], layout: SDLayout): + def __init__( + self, drive: Union[str, Path], layout: Union[SDLayout, ConfigSource] = "wirefree-sd-layout" + ): self.drive = drive - self.layout = layout + self.layout = SDLayout.from_any(layout) self.logger = init_logger("SDCard") # Private attributes used when the file reading context is entered diff --git a/miniscope_io/logging.py b/mio/logging.py similarity index 50% rename from miniscope_io/logging.py rename to mio/logging.py index 3fba3eb3..a7d71f06 100644 --- a/miniscope_io/logging.py +++ b/mio/logging.py @@ -3,13 +3,14 @@ """ import logging +import multiprocessing as mp from logging.handlers import RotatingFileHandler from pathlib import Path from typing import Optional, Union from rich.logging import RichHandler -from miniscope_io.models.config import LOG_LEVELS, Config +from mio.models.config import LOG_LEVELS, Config def init_logger( @@ -28,7 +29,7 @@ def init_logger( Args: name (str): Name of this logger. Ideally names are hierarchical - and indicate what they are logging for, eg. ``miniscope_io.sdcard`` + and indicate what they are logging for, eg. ``mio.sdcard`` and don't contain metadata like timestamps, etc. (which are in the logs) log_dir (:class:`pathlib.Path`): Directory to store file-based logs in. If ``None``, get from :class:`.Config`. If ``False`` , disable file logging. @@ -48,29 +49,92 @@ def init_logger( if log_dir is None: log_dir = config.log_dir if level is None: - level = config.logs.level_stdout + level: LOG_LEVELS = ( + config.logs.level_stdout if config.logs.level_stdout is not None else config.logs.level + ) if file_level is None: - file_level = config.logs.level_file + file_level: LOG_LEVELS = ( + config.logs.level_file if config.logs.level_file is not None else config.logs.level + ) if log_file_n is None: log_file_n = config.logs.file_n if log_file_size is None: log_file_size = config.logs.file_size - if not name.startswith("miniscope_io"): - name = "miniscope_io." + name + # set our logger to the minimum of the levels so that it always handles at least that severity + # even if one or the other handlers might not. + min_level = min([getattr(logging, level), getattr(logging, file_level)]) - logger = logging.getLogger(name) - logger.setLevel(level) + if not name.startswith("mio"): + name = "mio." + name - # Add handlers for stdout and file - if log_dir is not False: - logger.addHandler(_file_handler(name, file_level, log_dir, log_file_n, log_file_size)) + _init_root( + stdout_level=level, + file_level=file_level, + log_dir=log_dir, + log_file_n=log_file_n, + log_file_size=log_file_size, + ) - logger.addHandler(_rich_handler()) + logger = logging.getLogger(name) + logger.setLevel(min_level) + + # if run from a forked process, need to add different handlers to not collide + if mp.parent_process() is not None: + logger.addHandler( + _file_handler( + name=f"{name}_{mp.current_process().pid}", + file_level=file_level, + log_dir=log_dir, + log_file_n=log_file_n, + log_file_size=log_file_size, + ) + ) + logger.addHandler(_rich_handler(level)) + logger.propagate = False return logger +def _init_root( + stdout_level: LOG_LEVELS, + file_level: LOG_LEVELS, + log_dir: Path, + log_file_n: int = 5, + log_file_size: int = 2**22, +) -> None: + root_logger = logging.getLogger("mio") + file_handlers = [ + handler for handler in root_logger.handlers if isinstance(handler, RotatingFileHandler) + ] + stream_handlers = [ + handler for handler in root_logger.handlers if isinstance(handler, RichHandler) + ] + + if log_dir is not False and not file_handlers: + root_logger.addHandler( + _file_handler( + "mio", + file_level, + log_dir, + log_file_n, + log_file_size, + ) + ) + else: + for file_handler in file_handlers: + file_handler.setLevel(file_level) + + if not stream_handlers: + root_logger.addHandler(_rich_handler(stdout_level)) + else: + for stream_handler in stream_handlers: + stream_handler.setLevel(stdout_level) + + # prevent propagation to the default root + root_logger.propagate = False + + def _file_handler( name: str, file_level: LOG_LEVELS, @@ -90,11 +154,12 @@ def _file_handler( return file_handler -def _rich_handler() -> RichHandler: +def _rich_handler(level: LOG_LEVELS) -> RichHandler: rich_handler = RichHandler(rich_tracebacks=True, markup=True) rich_formatter = logging.Formatter( - "[bold green]\[%(name)s][/bold green] %(message)s", + r"[bold green]\[%(name)s][/bold green] %(message)s", datefmt="[%y-%m-%dT%H:%M:%S]", ) rich_handler.setFormatter(rich_formatter) + rich_handler.setLevel(level) return rich_handler diff --git a/miniscope_io/models/__init__.py b/mio/models/__init__.py similarity index 81% rename from miniscope_io/models/__init__.py rename to mio/models/__init__.py index 1b841a92..815b6156 100644 --- a/miniscope_io/models/__init__.py +++ b/mio/models/__init__.py @@ -2,13 +2,13 @@ Data models :) """ -from miniscope_io.models.models import ( +from mio.models.models import ( Container, MiniscopeConfig, MiniscopeIOModel, PipelineModel, ) -from miniscope_io.models.pipeline import ( +from mio.models.pipeline import ( Node, Pipeline, PipelineConfig, diff --git a/miniscope_io/models/buffer.py b/mio/models/buffer.py similarity index 92% rename from miniscope_io/models/buffer.py rename to mio/models/buffer.py index 49b177fc..3af7359c 100644 --- a/miniscope_io/models/buffer.py +++ b/mio/models/buffer.py @@ -6,10 +6,11 @@ from collections.abc import Sequence from typing import Type, TypeVar -from miniscope_io.models import Container, MiniscopeConfig +from mio.models import Container, MiniscopeConfig +from mio.models.mixins import ConfigYAMLMixin -class BufferHeaderFormat(MiniscopeConfig): +class BufferHeaderFormat(MiniscopeConfig, ConfigYAMLMixin): """ Format model used to parse header at the beginning of every buffer. @@ -86,7 +87,7 @@ def from_format( """ header_data = dict() - for hd, header_index in format.model_dump().items(): + for hd, header_index in format.model_dump(exclude=set(format.HEADER_FIELDS)).items(): if header_index is not None: header_data[hd] = vals[header_index] diff --git a/mio/models/config.py b/mio/models/config.py new file mode 100644 index 00000000..3884dc87 --- /dev/null +++ b/mio/models/config.py @@ -0,0 +1,259 @@ +""" +Module-global configuration models +""" + +from pathlib import Path +from typing import Any, Literal, Optional + +import yaml +from platformdirs import PlatformDirs +from pydantic import Field, TypeAdapter, field_validator, model_validator +from pydantic_settings import ( + BaseSettings, + PydanticBaseSettingsSource, + PyprojectTomlConfigSettingsSource, + SettingsConfigDict, + YamlConfigSettingsSource, +) + +from mio.models import MiniscopeIOModel +from mio.models.mixins import YAMLMixin + +_default_userdir = Path().home() / ".config" / "mio" +_dirs = PlatformDirs("mio", "mio") +_global_config_path = Path(_dirs.user_config_path) / "mio_config.yaml" +LOG_LEVELS = Literal["DEBUG", "INFO", "WARNING", "ERROR"] + + +class LogConfig(MiniscopeIOModel): + """ + Configuration for logging + """ + + level: LOG_LEVELS = "INFO" + """ + Severity of log messages to process. + """ + level_file: Optional[LOG_LEVELS] = None + """ + Severity for file-based logging. If unset, use ``level`` + """ + level_stdout: Optional[LOG_LEVELS] = None + """ + Severity for stream-based logging. If unset, use ``level`` + """ + file_n: int = 5 + """ + Number of log files to rotate through + """ + file_size: int = 2**22 # roughly 4MB + """ + Maximum size of log files (bytes) + """ + + @field_validator("level", "level_file", "level_stdout", mode="before") + @classmethod + def uppercase_levels(cls, value: Optional[str] = None) -> Optional[str]: + """ + Ensure log level strings are uppercased + """ + if value is not None: + value = value.upper() + return value + + +class Config(BaseSettings, YAMLMixin): + """ + Runtime configuration for mio. + + See https://docs.pydantic.dev/latest/concepts/pydantic_settings/ + + Set values either in an ``.env`` file or using environment variables + prefixed with ``MIO_*``. Values in nested models are separated with ``__`` , + eg. ``MIO_LOGS__LEVEL`` + + See ``.env.example`` in repository root + + Paths are set relative to ``user_dir`` by default, unless explicitly specified. + """ + + user_dir: Path = Field( + _global_config_path.parent, + description="Base directory to store user configuration and other temporary files, " + "other paths are relative to this by default", + ) + config_dir: Path = Field(Path("config"), description="Location to store user configs") + log_dir: Path = Field(Path("logs"), description="Location to store logs") + + logs: LogConfig = Field(LogConfig(), description="Additional settings for logs") + + @field_validator("user_dir", mode="before") + @classmethod + def folder_exists(cls, v: Path) -> Path: + """Ensure user_dir exists, make it otherwise""" + v = Path(v) + v.mkdir(exist_ok=True, parents=True) + + assert v.exists(), f"{v} does not exist!" + return v + + @model_validator(mode="after") + def paths_relative_to_basedir(self) -> "Config": + """If relative paths are given, make them absolute relative to ``user_dir``""" + paths = ("log_dir", "config_dir") + for path_name in paths: + path = getattr(self, path_name) # type: Path + if not path.is_absolute(): + path = self.user_dir / path + setattr(self, path_name, path) + path.mkdir(exist_ok=True) + assert path.exists() + return self + + model_config = SettingsConfigDict( + env_prefix="mio_", + env_file=".env", + env_nested_delimiter="__", + extra="ignore", + nested_model_default_partial_update=True, + yaml_file="mio_config.yaml", + pyproject_toml_table_header=("tool", "mio", "config"), + ) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + """ + Read config settings from, in order of priority from high to low, where + high priorities override lower priorities: + * in the arguments passed to the class constructor (not user configurable) + * in environment variables like ``export MIO_LOG_DIR=~/`` + * in a ``.env`` file in the working directory + * in a ``mio_config.yaml`` file in the working directory + * in the ``tool.mio.config`` table in a ``pyproject.toml`` file + in the working directory + * in a user ``mio_config.yaml`` file, configured by `user_dir` in any of the other sources + * in the global ``mio_config.yaml`` file in the platform-specific data directory + (use ``mio config get global_config`` to find its location) + * the default values in the :class:`.GlobalConfig` model + """ + _create_default_global_config() + + return ( + init_settings, + env_settings, + dotenv_settings, + YamlConfigSettingsSource(settings_cls), + PyprojectTomlConfigSettingsSource(settings_cls), + _UserYamlConfigSource(settings_cls), + YamlConfigSettingsSource(settings_cls, yaml_file=_global_config_path), + ) + + +def set_user_dir(path: Path) -> None: + """ + Set the location of the user dir in the global config file + """ + _update_value(_global_config_path, "user_dir", str(path)) + + +def _create_default_global_config(path: Path = _global_config_path, force: bool = False) -> None: + """ + Create a default global `mio_config.yaml` file. + + Args: + force (bool): Override any existing global config + """ + if path.exists() and not force: + return + + path.parent.mkdir(parents=True, exist_ok=True) + config = {"user_dir": str(path.parent)} + with open(path, "w") as f: + yaml.safe_dump(config, f) + + +class _UserYamlConfigSource(YamlConfigSettingsSource): + """ + Yaml config source that gets the location of the user settings file from the prior sources + """ + + def __init__(self, *args: Any, **kwargs: Any): + self._user_config = None + super().__init__(*args, **kwargs) + + @property + def user_config_path(self) -> Optional[Path]: + """ + Location of the user-level ``mio_config.yaml`` file, + given the current state of prior config sources, + including the global config file + """ + config_file = None + user_dir: Optional[str] = self.current_state.get("user_dir", None) + if user_dir is None: + # try and get from global config + if _global_config_path.exists(): + with open(_global_config_path) as f: + data = yaml.safe_load(f) + user_dir = data.get("user_dir", None) + + if user_dir is not None: + # handle .yml or .yaml + config_files = list(Path(user_dir).glob("mio_config.*")) + if len(config_files) != 0: + config_file = config_files[0] + + else: + # gotten from higher priority config sources + config_file = Path(user_dir) / "mio_config.yaml" + return config_file + + @property + def user_config(self) -> dict[str, Any]: + """ + Contents of the user config file + """ + if self._user_config is None: + if self.user_config_path is None or not self.user_config_path.exists(): + self._user_config = {} + else: + self._user_config = self._read_files(self.user_config_path) + + return self._user_config + + def __call__(self) -> dict[str, Any]: + return ( + TypeAdapter(dict[str, Any]).dump_python(self.user_config) + if self.nested_model_default_partial_update + else self.user_config + ) + + +def _update_value(path: Path, key: str, value: Any) -> None: + """ + Update a single value in a yaml file + + .. todo:: + + Make this work with nested keys + + """ + data = None + if path.exists(): + with open(path) as f: + data = yaml.safe_load(f) + + if data is None: + data = {} + + data[key] = value + + with open(path, "w") as f: + yaml.dump(data, f) diff --git a/miniscope_io/models/data.py b/mio/models/data.py similarity index 97% rename from miniscope_io/models/data.py rename to mio/models/data.py index cd358c54..da4e2da8 100644 --- a/miniscope_io/models/data.py +++ b/mio/models/data.py @@ -8,7 +8,7 @@ import pandas as pd from pydantic import BaseModel, field_validator -from miniscope_io.models.sdcard import SDBufferHeader +from mio.models.sdcard import SDBufferHeader class Frame(BaseModel, arbitrary_types_allowed=True): diff --git a/miniscope_io/models/devupdate.py b/mio/models/devupdate.py similarity index 66% rename from miniscope_io/models/devupdate.py rename to mio/models/devupdate.py index a432a838..23ce900c 100644 --- a/miniscope_io/models/devupdate.py +++ b/mio/models/devupdate.py @@ -19,10 +19,10 @@ class UpdateCommandDefinitions: Definitions of Bit masks and headers for remote update commands. """ - # Header to indicate target/value. + # Header to indicate key/value. # It probably won't be used in other places so defined here. id_header = 0b00000000 - target_header = 0b11000000 + key_header = 0b11000000 LSB_header = 0b01000000 MSB_header = 0b10000000 LSB_value_mask = 0b000000111111 # value below 12-bit @@ -30,18 +30,21 @@ class UpdateCommandDefinitions: reset_byte = 0b11111111 -class UpdateTarget(int, Enum): +class UpdateKey(int, Enum): """ - Targets to update. Needs to be under 6-bit. + Keys to update. Needs to be under 6-bit. """ LED = 0 GAIN = 1 ROI_X = 2 ROI_Y = 3 + SUBSAMPLE = 4 + """ ROI_WIDTH = 4 # not implemented ROI_HEIGHT = 5 # not implemented EWL = 6 # not implemented + """ DEVICE = 50 # for device commands @@ -52,7 +55,7 @@ class DevUpdateCommand(BaseModel): device_id: int port: str - target: UpdateTarget + key: UpdateKey value: int model_config = ConfigDict(arbitrary_types_allowed=True) @@ -60,23 +63,26 @@ class DevUpdateCommand(BaseModel): @model_validator(mode="after") def validate_values(cls, values: dict) -> dict: """ - Validate values based on target. + Validate values based on key. """ - target = values.target + key = values.key value = values.value - if target == UpdateTarget.LED: + if key == UpdateKey.LED: assert 0 <= value <= 100, "For LED, value must be between 0 and 100" - elif target == UpdateTarget.GAIN: + elif key == UpdateKey.GAIN: assert value in [1, 2, 4], "For GAIN, value must be 1, 2, or 4" - elif target == UpdateTarget.DEVICE: + elif key == UpdateKey.DEVICE: assert value in [DeviceCommand.REBOOT.value], "For DEVICE, value must be in [200]" - elif target in UpdateTarget: + elif key == UpdateKey.SUBSAMPLE: + assert value in [0, 1], "For SUBSAMPLE, value must be in [0, 1]" + elif key in [UpdateKey.ROI_X, UpdateKey.ROI_Y]: + # validation not implemented + pass + elif key in UpdateKey: raise NotImplementedError() else: - raise ValueError( - f"{target} is not a valid update target," "need an instance of UpdateTarget" - ) + raise ValueError(f"{key} is not a valid update key," "need an instance of UpdateKey") return values @field_validator("port") @@ -101,24 +107,24 @@ def validate_port(cls, value: str) -> str: raise ValueError(f"Port {value} not found") return value - @field_validator("target", mode="before") - def validate_target(cls, value: str) -> UpdateTarget: + @field_validator("key", mode="before") + def validate_key(cls, value: str) -> UpdateKey: """ - Validate and convert target string to UpdateTarget Enum type. + Validate and convert key string to UpdateKey Enum type. Args: - value (str): Target to validate. + value (str): Key to validate. Returns: - UpdateTarget: Validated target as UpdateTarget. + UpdateKey: Validated key as UpdateKey. Raises: - ValueError: If target not found. + ValueError: If key not found. """ try: - return UpdateTarget[value] + return UpdateKey[value] except KeyError as e: raise ValueError( - f"Target {value} not found, must be a member of UpdateTarget:" - f" {list(UpdateTarget.__members__.keys())}." + f"Key {value} not found, must be a member of UpdateKey:" + f" {list(UpdateKey.__members__.keys())}." ) from e diff --git a/mio/models/mixins.py b/mio/models/mixins.py new file mode 100644 index 00000000..fd09a240 --- /dev/null +++ b/mio/models/mixins.py @@ -0,0 +1,327 @@ +""" +Mixin classes that are to be used alongside specific models +to use composition for functionality and inheritance for semantics. +""" + +import re +import shutil +from importlib.metadata import version +from itertools import chain +from pathlib import Path +from typing import Any, ClassVar, List, Literal, Optional, Type, TypeVar, Union, overload + +import yaml +from pydantic import BaseModel, Field, ValidationError, field_validator + +from mio.types import ConfigID, ConfigSource, PythonIdentifier, valid_config_id + +T = TypeVar("T") + + +class YamlDumper(yaml.SafeDumper): + """Dumper that can represent extra types like Paths""" + + def represent_path(self, data: Path) -> yaml.ScalarNode: + """Represent a path as a string""" + return self.represent_scalar("tag:yaml.org,2002:str", str(data)) + + +YamlDumper.add_representer(type(Path()), YamlDumper.represent_path) + + +class YAMLMixin: + """ + Mixin class that provides :meth:`.from_yaml` and :meth:`.to_yaml` + classmethods + """ + + @classmethod + def from_yaml(cls: Type[T], file_path: Union[str, Path]) -> T: + """Instantiate this class by passing the contents of a yaml file as kwargs""" + with open(file_path) as file: + config_data = yaml.safe_load(file) + return cls(**config_data) + + def to_yaml(self, path: Optional[Path] = None, **kwargs: Any) -> str: + """ + Dump the contents of this class to a yaml file, returning the + contents of the dumped string + """ + data_str = self.to_yamls(**kwargs) + if path: + with open(path, "w") as file: + file.write(data_str) + + return data_str + + def to_yamls(self, **kwargs: Any) -> str: + """ + Dump the contents of this class to a yaml string + + Args: + **kwargs: passed to :meth:`.BaseModel.model_dump` + """ + data = self._dump_data(**kwargs) + return yaml.dump(data, Dumper=YamlDumper, sort_keys=False) + + def _dump_data(self, **kwargs: Any) -> dict: + data = self.model_dump(**kwargs) if isinstance(self, BaseModel) else self.__dict__ + return data + + +class ConfigYAMLMixin(BaseModel, YAMLMixin): + """ + Yaml Mixin class that always puts a header consisting of + + * `id` - unique identifier for this config + * `mio_model` - fully-qualified module path to model class + * `mio_version` - version of mio when this model was created + + at the top of the file. + """ + + id: ConfigID + mio_model: PythonIdentifier = Field(None, validate_default=True) + mio_version: str = version("mio") + + HEADER_FIELDS: ClassVar[tuple[str]] = ("id", "mio_model", "mio_version") + + @classmethod + def from_yaml(cls: Type[T], file_path: Union[str, Path]) -> T: + """Instantiate this class by passing the contents of a yaml file as kwargs""" + with open(file_path) as file: + config_data = yaml.safe_load(file) + + # fill in any missing fields in the source file needed for a header + config_data = cls._complete_header(config_data, file_path) + try: + instance = cls(**config_data) + except ValidationError: + if (backup_path := file_path.with_suffix(".yaml.bak")).exists(): + from mio.logging import init_logger + + init_logger("config").debug( + f"Model instantiation failed, restoring modified backup from {backup_path}..." + ) + shutil.copy(backup_path, file_path) + raise + + return instance + + @classmethod + def from_id(cls: Type[T], id: ConfigID) -> T: + """ + Instantiate a model from a config `id` specified in one of the .yaml configs in + either the user :attr:`.Config.config_dir` or the packaged ``config`` dir. + + .. note:: + + this method does not yet validate that the config matches the model loading it + + """ + globs = [src.rglob("*.y*ml") for src in cls.config_sources] + for config_file in chain(*globs): + try: + file_id = yaml_peek("id", config_file) + if file_id == id: + from mio.logging import init_logger + + init_logger("config").debug( + "Model for %s found at %s", cls._model_name(), config_file + ) + return cls.from_yaml(config_file) + except KeyError: + continue + + from mio import Config + + raise KeyError(f"No config with id {id} found in {Config().config_dir}") + + @classmethod + def from_any(cls: Type[T], source: Union[ConfigSource, T]) -> T: + """ + Try and instantiate a config model from any supported constructor. + + Args: + source (:class:`.ConfigID`, :class:`.Path`, :class:`.PathLike[str]`): + Either + + * the ``id`` of a config file in the user configs directory or builtin + * a relative ``Path`` to a config file, relative to the current working directory + * a relative ``Path`` to a config file, relative to the user config directory + * an absolute ``Path`` to a config file + * an instance of the class to be constructed (returned unchanged) + + """ + if isinstance(source, cls): + return source + elif valid_config_id(source): + return cls.from_id(source) + else: + from mio import Config + + source = Path(source) + if source.suffix in (".yaml", ".yml"): + if source.exists(): + # either relative to cwd or absolute + return cls.from_yaml(source) + elif ( + not source.is_absolute() + and (user_source := Config().config_dir / source).exists() + ): + return cls.from_yaml(user_source) + + raise ValueError( + f"Instance of config model {cls.__name__} could not be instantiated from " + f"{source} - id or file not found, or type not supported" + ) + + @field_validator("mio_model", mode="before") + @classmethod + def fill_mio_model(cls, v: Optional[str]) -> PythonIdentifier: + """Get name of instantiating model, if not provided""" + if v is None: + v = cls._model_name() + return v + + @classmethod + @property + def config_sources(cls: Type[T]) -> List[Path]: + """ + Directories to search for config files, in order of priority + such that earlier sources are preferred over later sources. + """ + from mio import CONFIG_DIR, Config + + return [Config().config_dir, CONFIG_DIR] + + def _dump_data(self, **kwargs: Any) -> dict: + """Ensure that header is prepended to model data""" + return {**self._yaml_header(self), **super()._dump_data(**kwargs)} + + @classmethod + def _model_name(cls) -> PythonIdentifier: + return f"{cls.__module__}.{cls.__name__}" + + @classmethod + def _yaml_header(cls, instance: Union[T, dict]) -> dict: + if isinstance(instance, dict): + model_id = instance.get("id", None) + mio_model = instance.get("mio_model", cls._model_name()) + mio_version = instance.get("mio_version", version("mio")) + else: + model_id = getattr(instance, "id", None) + mio_model = getattr(instance, "mio_model", cls._model_name()) + mio_version = getattr(instance, "mio_version", version("mio")) + + if model_id is None: + # if missing an id, try and recover with model default cautiously + # so we throw the exception during validation and not here, for clarity. + model_id = getattr(cls.model_fields.get("id", None), "default", None) + if type(model_id).__name__ == "PydanticUndefinedType": + model_id = None + + return { + "id": model_id, + "mio_model": mio_model, + "mio_version": mio_version, + } + + @classmethod + def _complete_header(cls: Type[T], data: dict, file_path: Union[str, Path]) -> dict: + """fill in any missing fields in the source file needed for a header""" + + missing_fields = set(cls.HEADER_FIELDS) - set(data.keys()) + keys = tuple(data.keys()) + out_of_order = len(keys) >= 3 and keys[0:3] != cls.HEADER_FIELDS + + if missing_fields or out_of_order: + if missing_fields: + msg = f"Missing required header fields {missing_fields} in config model " + f"{str(file_path)}. Updating file (preserving backup)..." + else: + msg = f"Header keys were present, but either not at the start of {str(file_path)} " + "or in out of order. Updating file (preserving backup)..." + from mio import CONFIG_DIR + from mio.logging import init_logger + + logger = init_logger(cls.__name__) + logger.warning(msg) + logger.debug(data) + + header = cls._yaml_header(data) + data = {**header, **data} + if CONFIG_DIR not in file_path.parents: + shutil.copy(file_path, file_path.with_suffix(".yaml.bak")) + with open(file_path, "w") as yfile: + yaml.dump(data, yfile, Dumper=YamlDumper, sort_keys=False) + + return data + + +@overload +def yaml_peek( + key: str, path: Union[str, Path], root: bool = True, first: Literal[True] = True +) -> str: ... + + +@overload +def yaml_peek( + key: str, path: Union[str, Path], root: bool = True, first: Literal[False] = False +) -> List[str]: ... + + +@overload +def yaml_peek( + key: str, path: Union[str, Path], root: bool = True, first: bool = True +) -> Union[str, List[str]]: ... + + +def yaml_peek( + key: str, path: Union[str, Path], root: bool = True, first: bool = True +) -> Union[str, List[str]]: + """ + Peek into a yaml file without parsing the whole file to retrieve the value of a single key. + + This function is _not_ designed for robustness to the yaml spec, it is for simple key: value + pairs, not fancy shit like multiline strings, tagged values, etc. If you want it to be, + then i'm afraid you'll have to make a PR about it. + + Returns a string no matter what the yaml type is so ya have to do your own casting if you want + + Args: + key (str): The key to peek for + path (:class:`pathlib.Path` , str): The yaml file to peek into + root (bool): Only find keys at the root of the document (default ``True`` ), otherwise + find keys at any level of nesting. + first (bool): Only return the first appearance of the key (default). Otherwise return a + list of values (not implemented lol) + + Returns: + str + """ + if root: + pattern = re.compile( + rf"^(?P{key}):\s*\"*\'*(?P\S.*?)\"*\'*$", flags=re.MULTILINE + ) + else: + pattern = re.compile( + rf"^\s*(?P{key}):\s*\"*\'*(?P\S.*?)\"*\'*$", flags=re.MULTILINE + ) + + res = None + if first: + with open(path) as yfile: + for line in yfile: + res = pattern.match(line) + if res: + break + if res: + return res.groupdict()["value"] + else: + with open(path) as yfile: + text = yfile.read() + res = [match.groupdict()["value"] for match in pattern.finditer(text)] + if res: + return res + raise KeyError(f"Key {key} not found in {path}") diff --git a/miniscope_io/models/models.py b/mio/models/models.py similarity index 96% rename from miniscope_io/models/models.py rename to mio/models/models.py index 70b287dd..4c6773e7 100644 --- a/miniscope_io/models/models.py +++ b/mio/models/models.py @@ -14,7 +14,7 @@ class MiniscopeIOModel(BaseModel): """ - Root model for all miniscope_io models + Root model for all mio models """ diff --git a/miniscope_io/models/pipeline.py b/mio/models/pipeline.py similarity index 100% rename from miniscope_io/models/pipeline.py rename to mio/models/pipeline.py diff --git a/miniscope_io/models/sdcard.py b/mio/models/sdcard.py similarity index 92% rename from miniscope_io/models/sdcard.py rename to mio/models/sdcard.py index c6ad9395..178efe6b 100644 --- a/miniscope_io/models/sdcard.py +++ b/mio/models/sdcard.py @@ -6,8 +6,9 @@ from typing import Optional -from miniscope_io.models import MiniscopeConfig -from miniscope_io.models.buffer import BufferHeader, BufferHeaderFormat +from mio.models import MiniscopeConfig +from mio.models.buffer import BufferHeader, BufferHeaderFormat +from mio.models.mixins import ConfigYAMLMixin class SectorConfig(MiniscopeConfig): @@ -91,6 +92,8 @@ class SDBufferHeaderFormat(BufferHeaderFormat): Positions in the header for each frame """ + id: str = "sd-buffer-header" + length: int = 0 linked_list: int = 1 frame_num: int = 2 @@ -104,7 +107,7 @@ class SDBufferHeaderFormat(BufferHeaderFormat): battery_voltage: Optional[int] = None -class SDLayout(MiniscopeConfig): +class SDLayout(MiniscopeConfig, ConfigYAMLMixin): """ Data layout of an SD Card. @@ -131,12 +134,6 @@ class SDLayout(MiniscopeConfig): config: ConfigPositions = ConfigPositions() buffer: SDBufferHeaderFormat = SDBufferHeaderFormat() - version: Optional[str] = None - """ - Not Implemented: version stored in the SD card header that indicates - when this layout should be used - """ - class SDConfig(MiniscopeConfig): """ diff --git a/miniscope_io/models/sinks.py b/mio/models/sinks.py similarity index 76% rename from miniscope_io/models/sinks.py rename to mio/models/sinks.py index ca946d96..bfa48561 100644 --- a/miniscope_io/models/sinks.py +++ b/mio/models/sinks.py @@ -4,12 +4,12 @@ from pydantic import Field -from miniscope_io.models import MiniscopeConfig +from mio.models import MiniscopeConfig class StreamPlotterConfig(MiniscopeConfig): """ - Configuration for :class:`miniscope_io.plots.headers.StreamPlotter` + Configuration for :class:`mio.plots.headers.StreamPlotter` """ keys: list[str] = Field( @@ -27,7 +27,7 @@ class StreamPlotterConfig(MiniscopeConfig): class CSVWriterConfig(MiniscopeConfig): """ - Configuration for :class:`miniscope_io.io.BufferedCSVWriter` + Configuration for :class:`mio.io.BufferedCSVWriter` """ buffer: int = Field( diff --git a/miniscope_io/models/stream.py b/mio/models/stream.py similarity index 95% rename from miniscope_io/models/stream.py rename to mio/models/stream.py index 8aec738a..943fc84d 100644 --- a/miniscope_io/models/stream.py +++ b/mio/models/stream.py @@ -1,5 +1,5 @@ """ -Models for :mod:`miniscope_io.stream_daq` +Models for :mod:`mio.stream_daq` """ from pathlib import Path @@ -7,11 +7,11 @@ from pydantic import Field, computed_field, field_validator -from miniscope_io import DATA_DIR -from miniscope_io.models import MiniscopeConfig -from miniscope_io.models.buffer import BufferHeader, BufferHeaderFormat -from miniscope_io.models.mixins import YAMLMixin -from miniscope_io.models.sinks import CSVWriterConfig, StreamPlotterConfig +from mio import DATA_DIR +from mio.models import MiniscopeConfig +from mio.models.buffer import BufferHeader, BufferHeaderFormat +from mio.models.mixins import ConfigYAMLMixin +from mio.models.sinks import CSVWriterConfig, StreamPlotterConfig class ADCScaling(MiniscopeConfig): @@ -64,7 +64,7 @@ def scale_input_voltage(self, voltage_raw: float) -> float: class StreamBufferHeaderFormat(BufferHeaderFormat): """ Refinements of :class:`.BufferHeaderFormat` for - :class:`~miniscope_io.stream_daq.StreamDaq` + :class:`~mio.stream_daq.StreamDaq` Parameters ---------- @@ -86,7 +86,7 @@ class StreamBufferHeaderFormat(BufferHeaderFormat): class StreamBufferHeader(BufferHeader): """ Refinements of :class:`.BufferHeader` for - :class:`~miniscope_io.stream_daq.StreamDaq` + :class:`~mio.stream_daq.StreamDaq` """ pixel_count: int @@ -146,6 +146,10 @@ class StreamDevRuntime(MiniscopeConfig): 5, description="Buffer length for storing images in streamDaq", ) + queue_put_timeout: int = Field( + 5, + description="Timeout for putting data into the queue", + ) plot: Optional[StreamPlotterConfig] = Field( StreamPlotterConfig( keys=["timestamp", "buffer_count", "frame_buffer_count"], update_ms=1000, history=500 @@ -164,7 +168,7 @@ class StreamDevRuntime(MiniscopeConfig): ) -class StreamDevConfig(MiniscopeConfig, YAMLMixin): +class StreamDevConfig(MiniscopeConfig, ConfigYAMLMixin): """ Format model used to parse DAQ configuration yaml file (examples are in ./config) The model attributes are key-value pairs needed for reconstructing frames from data streams. diff --git a/miniscope_io/plots/__init__.py b/mio/plots/__init__.py similarity index 100% rename from miniscope_io/plots/__init__.py rename to mio/plots/__init__.py diff --git a/miniscope_io/plots/headers.py b/mio/plots/headers.py similarity index 89% rename from miniscope_io/plots/headers.py rename to mio/plots/headers.py index 71ba1d28..3e11c1c2 100644 --- a/miniscope_io/plots/headers.py +++ b/mio/plots/headers.py @@ -10,18 +10,15 @@ import numpy as np import pandas as pd -from miniscope_io.models.stream import StreamBufferHeader +from mio.models.stream import StreamBufferHeader try: import matplotlib.pyplot as plt -except ImportError as e: - raise ImportError( - "matplotlib is not a required dependency of miniscope-io, " - "install it with the miniscope-io[plot] extra or manually in your environment :)" - ) from e +except ImportError: + plt = None -def buffer_count(headers: pd.DataFrame, ax: plt.Axes) -> plt.Axes: +def buffer_count(headers: pd.DataFrame, ax: "plt.Axes") -> "plt.Axes": """ Plot number of buffers by time """ @@ -35,7 +32,7 @@ def buffer_count(headers: pd.DataFrame, ax: plt.Axes) -> plt.Axes: return ax -def dropped_buffers(headers: pd.DataFrame, ax: plt.Axes) -> plt.Axes: +def dropped_buffers(headers: pd.DataFrame, ax: "plt.Axes") -> "plt.Axes": """ Plot number of buffers by time """ @@ -45,7 +42,7 @@ def dropped_buffers(headers: pd.DataFrame, ax: plt.Axes) -> plt.Axes: return ax -def timestamps(headers: pd.DataFrame, ax: plt.Axes) -> plt.Axes: +def timestamps(headers: pd.DataFrame, ax: "plt.Axes") -> "plt.Axes": """ Plot frame number against time """ @@ -60,7 +57,7 @@ def timestamps(headers: pd.DataFrame, ax: plt.Axes) -> plt.Axes: return ax -def battery_voltage(headers: pd.DataFrame, ax: plt.Axes) -> plt.Axes: +def battery_voltage(headers: pd.DataFrame, ax: "plt.Axes") -> "plt.Axes": """ Plot battery voltage against time """ @@ -73,7 +70,7 @@ def battery_voltage(headers: pd.DataFrame, ax: plt.Axes) -> plt.Axes: def plot_headers( headers: pd.DataFrame, size: Optional[Tuple[int, int]] = None -) -> (plt.Figure, plt.Axes): +) -> ("plt.Figure", "plt.Axes"): """ Plot the headers (generated from :meth:`.Frame.to_df` ) @@ -132,6 +129,13 @@ def __init__( history_length (int): Number of headers to plot update_ms (int): milliseconds between each plot update """ + global plt + if plt is None: + raise ModuleNotFoundError( + "matplotlib is not a required dependency of mio, to use it, " + "install it manually or install mio with `pip install mio[plot]`" + ) + # If a single string is provided, convert it to a list with one element if isinstance(header_keys, str): header_keys = [header_keys] @@ -147,7 +151,7 @@ def __init__( def _init_plot( self, - ) -> tuple[plt.Figure, dict[str, plt.Axes], dict[str, plt.Line2D]]: + ) -> tuple["plt.Figure", dict[str, "plt.Axes"], dict[str, "plt.Line2D"]]: # initialize matplotlib plt.ion() diff --git a/miniscope_io/sources/mocks.py b/mio/sources/mocks.py similarity index 83% rename from miniscope_io/sources/mocks.py rename to mio/sources/mocks.py index 453f38d9..52945bba 100644 --- a/miniscope_io/sources/mocks.py +++ b/mio/sources/mocks.py @@ -4,7 +4,7 @@ Used in testing, but kept in-package since for now some sources need modifications to their source (and we can't import from tests) -Not to be considered part of the public interface of miniscope-io <3 +Not to be considered part of the public interface of mio <3 """ # ruff: noqa: D102 @@ -13,12 +13,16 @@ from pathlib import Path from typing import Dict, Optional -from miniscope_io.exceptions import EndOfRecordingException +from mio.exceptions import EndOfRecordingException class okDevMock: """ +<<<<<<<< HEAD:miniscope_io/sources/mocks.py Mock class for :class:`~miniscope_io.sources.opalkelly.okDev` +======== + Mock class for :class:`~mio.devices.opalkelly.okDev` +>>>>>>>> main:mio/devices/mocks.py """ DATA_FILE: Optional[Path] = None @@ -56,11 +60,11 @@ def __init__(self, serial_id: str = ""): with open(self.DATA_FILE, "rb") as dfile: self._buffer = bytearray(dfile.read()) - def uploadBit(self, bit_file: str) -> None: + def upload_bit(self, bit_file: str) -> None: assert Path(bit_file).exists() self.bit_file = Path(bit_file) - def readData(self, length: int, addr: int = 0xA0, blockSize: int = 16) -> bytearray: + def read_data(self, length: int, addr: int = 0xA0, blockSize: int = 16) -> bytearray: if self._buffer_position >= len(self._buffer): # Error if called after we have returned the last data raise EndOfRecordingException("End of sample buffer") @@ -70,5 +74,5 @@ def readData(self, length: int, addr: int = 0xA0, blockSize: int = 16) -> bytear self._buffer_position = end_pos return data - def setWire(self, addr: int, val: int) -> None: + def set_wire(self, addr: int, val: int) -> None: self._wires[addr] = val diff --git a/miniscope_io/sources/opalkelly.py b/mio/sources/opalkelly.py similarity index 87% rename from miniscope_io/sources/opalkelly.py rename to mio/sources/opalkelly.py index 108abdad..24331d60 100644 --- a/miniscope_io/sources/opalkelly.py +++ b/mio/sources/opalkelly.py @@ -2,13 +2,13 @@ Interfaces for OpalKelly (model number?) FPGAs """ -from miniscope_io.exceptions import ( +from mio.exceptions import ( DeviceConfigurationError, DeviceOpenError, StreamReadError, ) -from miniscope_io.logging import init_logger -from miniscope_io.vendor import opalkelly as ok +from mio.logging import init_logger +from mio.vendor import opalkelly as ok class okDev(ok.okCFrontPanel): @@ -33,7 +33,7 @@ def __init__(self, serial_id: str = ""): if ret == self.NoError: self.logger.info(f"Connected to {self.info.productName}") - def uploadBit(self, bit_file: str) -> None: + def upload_bit(self, bit_file: str) -> None: """ Upload a configuration bitfile to the FPGA @@ -51,7 +51,7 @@ def uploadBit(self, bit_file: str) -> None: ) ret = self.ResetFPGA() - def readData(self, length: int, addr: int = 0xA0, blockSize: int = 16) -> bytearray: + def read_data(self, length: int, addr: int = 0xA0, blockSize: int = 16) -> bytearray: """ Read a buffer's worth of data @@ -73,7 +73,7 @@ def readData(self, length: int, addr: int = 0xA0, blockSize: int = 16) -> bytear self.logger.warning(f"Only {ret} bytes read") return buf - def setWire(self, addr: int, val: int) -> None: + def set_wire(self, addr: int, val: int) -> None: """ .. todo:: diff --git a/miniscope_io/stream_daq.py b/mio/stream_daq.py similarity index 78% rename from miniscope_io/stream_daq.py rename to mio/stream_daq.py index 25266f7d..6ca7cf38 100644 --- a/miniscope_io/stream_daq.py +++ b/mio/stream_daq.py @@ -5,6 +5,7 @@ import logging import multiprocessing import os +import queue import sys import time from pathlib import Path @@ -15,25 +16,23 @@ import serial from bitstring import BitArray, Bits -from miniscope_io import init_logger -from miniscope_io.bit_operation import BufferFormatter -from miniscope_io.exceptions import EndOfRecordingException, StreamReadError -from miniscope_io.formats.stream import StreamBufferHeader as StreamBufferHeaderFormat -from miniscope_io.io import BufferedCSVWriter -from miniscope_io.models.stream import ( +from mio import init_logger +from mio.bit_operation import BufferFormatter +from mio.exceptions import EndOfRecordingException, StreamReadError +from mio.io import BufferedCSVWriter +from mio.models.stream import ( StreamBufferHeader, + StreamBufferHeaderFormat, StreamDevConfig, ) -from miniscope_io.models.stream import ( - StreamBufferHeaderFormat as StreamBufferHeaderFormatType, -) -from miniscope_io.plots.headers import StreamPlotter -from miniscope_io.sources.mocks import okDevMock +from mio.plots.headers import StreamPlotter +from mio.sources.mocks import okDevMock +from mio.types import ConfigSource HAVE_OK = False ok_error = None try: - from miniscope_io.sources.opalkelly import okDev + from mio.sources.opalkelly import okDev HAVE_OK = True except (ImportError, ModuleNotFoundError): @@ -64,13 +63,13 @@ class StreamDaq: Supported sources and required inputs are described in StreamDevConfig model documentation. This function's entry point is the main function, which should be used from the stream_image_capture command installed with the package. - Example configuration yaml files are stored in /miniscope-io/config/. + Example configuration yaml files are stored in /mio/config/. Examples -------- $ mio stream capture -c path/to/config.yml -o output_filename.avi Connected to XEM7310-A75 - Succesfully uploaded /miniscope-io/miniscope_io/sources/selected_bitfile.bit + Succesfully uploaded /mio/data/bitfile/selected_bitfile.bit FrontPanel is supported .. todo:: @@ -81,8 +80,8 @@ class StreamDaq: def __init__( self, - device_config: Union[StreamDevConfig, Path], - header_fmt: StreamBufferHeaderFormatType = StreamBufferHeaderFormat, + device_config: Union[StreamDevConfig, ConfigSource], + header_fmt: Union[StreamBufferHeaderFormat, ConfigSource] = "stream-buffer-header", ) -> None: """ Constructer for the class. @@ -92,19 +91,25 @@ def __init__( ---------- config : StreamDevConfig | Path DAQ configurations imported from the input yaml file. - Examples and required properties can be found in /miniscope-io/config/example.yml + Examples and required properties can be found in /mio/config/example.yml Passed either as the instantiated config object or a path to on-disk yaml configuration header_fmt : MetadataHeaderFormat, optional Header format used to parse information from buffer header, by default `MetadataHeaderFormat()`. """ - if isinstance(device_config, (str, Path)): - device_config = StreamDevConfig.from_yaml(device_config) self.logger = init_logger("streamDaq") - self.config = device_config - self.header_fmt = header_fmt + self.config = StreamDevConfig.from_any(device_config) + self.header_fmt = StreamBufferHeaderFormat.from_any(header_fmt) + if isinstance(header_fmt, str): + self.header_fmt = StreamBufferHeaderFormat.from_id(header_fmt) + elif isinstance(header_fmt, StreamBufferHeaderFormat): + self.header_fmt = header_fmt + else: + raise TypeError( + "header_fmt should be an instance of StreamBufferHeaderFormat or a config ID." + ) self.preamble = self.config.preamble self.terminate: multiprocessing.Event = multiprocessing.Event() @@ -163,7 +168,9 @@ def _parse_header(self, buffer: bytes) -> Tuple[StreamBufferHeader, np.ndarray]: reverse_payload_bytes=self.config.reverse_payload_bytes, ) - header_data = StreamBufferHeader.from_format(header, self.header_fmt, construct=True) + header_data = StreamBufferHeader.from_format( + header.astype(int), self.header_fmt, construct=True + ) header_data.adc_scaling = self.config.adc_scale return header_data, payload @@ -235,11 +242,16 @@ def _uart_recv( log_uart_buffer = BitArray([x for x in uart_bites]) try: - while 1: + while not self.terminate.is_set(): # read UART data until preamble and put into queue uart_bites = serial_port.read_until(pre_bytes) log_uart_buffer = [x for x in uart_bites] - serial_buffer_queue.put(log_uart_buffer) + try: + serial_buffer_queue.put( + log_uart_buffer, block=True, timeout=self.config.runtime.queue_put_timeout + ) + except queue.Full: + self.logger.warning("Serial buffer queue full, skipping buffer.") finally: time.sleep(1) # time for ending other process serial_port.close() @@ -253,13 +265,13 @@ def _init_okdev(self, BIT_FILE: Path) -> Union[okDev, okDevMock]: else: dev = okDev() - dev.uploadBit(str(BIT_FILE)) - dev.setWire(0x00, 0b0010) + dev.upload_bit(str(BIT_FILE)) + dev.set_wire(0x00, 0b0010) time.sleep(0.01) - dev.setWire(0x00, 0b0) - dev.setWire(0x00, 0b1000) + dev.set_wire(0x00, 0b0) + dev.set_wire(0x00, 0b1000) time.sleep(0.01) - dev.setWire(0x00, 0b0) + dev.set_wire(0x00, 0b0) return dev def _fpga_recv( @@ -322,12 +334,16 @@ def _fpga_recv( locallogs.debug("Starting capture") try: - while not self.terminate.is_set(): + while 1: try: - buf = dev.readData(read_length) - except (EndOfRecordingException, StreamReadError, KeyboardInterrupt): + buf = dev.read_data(read_length) + except (EndOfRecordingException, KeyboardInterrupt): locallogs.debug("Got end of recording exception, breaking") break + except StreamReadError: + locallogs.exception("Read failed, continuing") + # It might be better to choose continue or break with a continuous flag + continue if capture_binary: with open(capture_binary, "ab") as file: @@ -342,13 +358,25 @@ def _fpga_recv( buf_start + len(self.preamble), buf_stop + len(self.preamble), ) - serial_buffer_queue.put(cur_buffer[buf_start:buf_stop].tobytes()) + try: + serial_buffer_queue.put( + cur_buffer[buf_start:buf_stop].tobytes(), + block=True, + timeout=self.config.runtime.queue_put_timeout, + ) + except queue.Full: + locallogs.warning("Serial buffer queue full, skipping buffer.") if pre_pos: cur_buffer = cur_buffer[pre_pos[-1] :] finally: locallogs.debug("Quitting, putting sentinel in queue") - serial_buffer_queue.put(None) + try: + serial_buffer_queue.put( + None, block=True, timeout=self.config.runtime.queue_put_timeout + ) + except queue.Full: + locallogs.error("Serial buffer queue full, Could not put sentinel.") def _buffer_to_frame( self, @@ -381,7 +409,6 @@ def _buffer_to_frame( try: for serial_buffer in exact_iter(serial_buffer_queue.get, None): - header_data, serial_buffer = self._parse_header(serial_buffer) header_list.append(header_data) @@ -401,7 +428,14 @@ def _buffer_to_frame( f"Discarding buffer." ) if header_list: - frame_buffer_queue.put((None, header_list)) + try: + frame_buffer_queue.put( + (None, header_list), + block=True, + timeout=self.config.runtime.queue_put_timeout, + ) + except queue.Full: + locallogs.warning("Frame buffer queue full, skipping frame.") continue # if first buffer of a frame @@ -411,7 +445,14 @@ def _buffer_to_frame( continue # push previous frame_buffer into frame_buffer queue - frame_buffer_queue.put((frame_buffer, header_list)) + try: + frame_buffer_queue.put( + (frame_buffer, header_list), + block=True, + timeout=self.config.runtime.queue_put_timeout, + ) + except queue.Full: + locallogs.warning("Frame buffer queue full, skipping frame.") # init new frame_buffer frame_buffer = frame_buffer_prealloc.copy() @@ -434,11 +475,22 @@ def _buffer_to_frame( locallogs.debug( "----buffer #" + str(header_data.frame_buffer_count) + " stored" ) - finally: - frame_buffer_queue.put((None, header_list)) # for getting remaining buffers. - locallogs.debug("Quitting, putting sentinel in queue") - frame_buffer_queue.put(None) + try: + # get remaining buffers. + frame_buffer_queue.put( + (None, header_list), block=True, timeout=self.config.runtime.queue_put_timeout + ) + except queue.Full: + locallogs.warning("Frame buffer queue full, skipping frame.") + + try: + frame_buffer_queue.put( + None, block=True, timeout=self.config.runtime.queue_put_timeout + ) + locallogs.debug("Quitting, putting sentinel in queue") + except queue.Full: + locallogs.error("Frame buffer queue full, Could not put sentinel.") def _format_frame( self, @@ -468,11 +520,15 @@ def _format_frame( try: for frame_data, header_list in exact_iter(frame_buffer_queue.get, None): - if not frame_data: - imagearray.put((None, header_list)) - continue - if len(frame_data) == 0: - imagearray.put((None, header_list)) + if not frame_data or len(frame_data) == 0: + try: + imagearray.put( + (None, header_list), + block=True, + timeout=self.config.runtime.queue_put_timeout, + ) + except queue.Full: + locallogs.warning("Image array queue full, skipping frame.") continue frame_data = np.concatenate(frame_data, axis=0) @@ -484,7 +540,8 @@ def _format_frame( expected_size = self.config.frame_width * self.config.frame_height provided_size = frame_data.size locallogs.exception( - "Frame size doesn't match: %s. Expected size: %d, got size: %d elements. " + "Frame size doesn't match: %s. " + " Expected size: %d, got size: %d." "Replacing with zeros.", e, expected_size, @@ -493,10 +550,20 @@ def _format_frame( frame = np.zeros( (self.config.frame_width, self.config.frame_height), dtype=np.uint8 ) - imagearray.put((frame, header_list)) + try: + imagearray.put( + (frame, header_list), + block=True, + timeout=self.config.runtime.queue_put_timeout, + ) + except queue.Full: + locallogs.warning("Image array queue full, skipping frame.") finally: locallogs.debug("Quitting, putting sentinel in queue") - imagearray.put(None) + try: + imagearray.put(None, block=True, timeout=self.config.runtime.queue_put_timeout) + except queue.Full: + locallogs.error("Image array queue full, Could not put sentinel.") def init_video( self, path: Union[Path, str], fourcc: str = "Y800", **kwargs: dict @@ -528,6 +595,19 @@ def init_video( out = cv2.VideoWriter(str(path), fourcc, frame_rate, frame_size, **kwargs) return out + def alive_processes(self) -> List[multiprocessing.Process]: + """ + Return a list of alive processes. + + Returns + ------- + List[multiprocessing.Process] + List of alive processes. + """ + + raise NotImplementedError("Not implemented yet") + return None + def capture( self, source: Literal["uart", "fpga"], @@ -547,7 +627,7 @@ def capture( source : Literal[uart, fpga] Device source. read_length : Optional[int], optional - Passed to :func:`~miniscope_io.stream_daq.stream_daq._fpga_recv` when + Passed to :func:`~mio.stream_daq.stream_daq._fpga_recv` when `source == "fpga"`, by default None. video: Path, optional If present, a path to an output video file @@ -638,7 +718,9 @@ def capture( self._buffered_writer = BufferedCSVWriter( metadata, buffer_size=self.config.runtime.csvwriter.buffer ) - self._buffered_writer.append(list(StreamBufferHeader.model_fields.keys())) + self._buffered_writer.append( + list(StreamBufferHeader.model_fields.keys()) + ["unix_time"] + ) try: for image, header_list in exact_iter(imagearray.get, None): @@ -650,7 +732,6 @@ def capture( show_metadata=show_metadata, metadata=metadata, ) - except KeyboardInterrupt: self.logger.exception( "Quitting capture, processing remaining frames. Ctrl+C again to force quit" @@ -711,12 +792,6 @@ def _handle_frame( Further refactor to break into smaller pieces, not have to pass 100 args every time. """ - if show_video is True: - cv2.imshow("image", image) - cv2.waitKey(1) - if writer: - picture = cv2.cvtColor(image, cv2.COLOR_GRAY2BGR) # If your image is grayscale - writer.write(picture) if show_metadata or metadata: for header in header_list: if show_metadata: @@ -728,9 +803,26 @@ def _handle_frame( if metadata: self.logger.debug("Saving header metadata") try: - self._buffered_writer.append(list(header.model_dump().values())) + self._buffered_writer.append( + list(header.model_dump(warnings=False).values()) + [time.time()] + ) except Exception as e: self.logger.exception(f"Exception saving headers: \n{e}") + if image is None or image.size == 0: + self.logger.warning("Empty frame received, skipping.") + return + if show_video: + try: + cv2.imshow("image", image) + cv2.waitKey(1) + except cv2.error as e: + self.logger.exception(f"Error displaying frame: {e}") + if writer: + try: + picture = cv2.cvtColor(image, cv2.COLOR_GRAY2BGR) # If your image is grayscale + writer.write(picture) + except cv2.error as e: + self.logger.exception(f"Exception writing frame: {e}") # DEPRECATION: v0.3.0 diff --git a/mio/types.py b/mio/types.py new file mode 100644 index 00000000..c78350da --- /dev/null +++ b/mio/types.py @@ -0,0 +1,93 @@ +""" +Type and type annotations +""" + +import re +import sys +from os import PathLike +from pathlib import Path +from typing import Annotated, Any, Tuple, Union, NamedTuple, Tuple, Union + +from pydantic import AfterValidator, Field + +if sys.version_info < (3, 10): + from typing_extensions import TypeAlias, TypeIs +elif sys.version_info < (3, 13): + from typing import TypeAlias + + from typing_extensions import TypeIs +else: + from typing import TypeAlias, TypeIs + +CONFIG_ID_PATTERN = r"[\w\-\/#]+" +""" +Any alphanumeric string (\w), as well as +- ``-`` +- ``/`` +- ``#`` +(to allow hierarchical IDs as well as fragment IDs). + +Specficially excludes ``.`` to avoid confusion between IDs, paths, and python module names + +May be made less restrictive in the future, will not be made more restrictive. +""" + + +def _is_identifier(val: str) -> str: + # private validation method to validate the parts of a fully-qualified python identifier + # defined first and not made public bc used as a validator, + # distinct from a boolean "is_{x}" check + + for part in val.split("."): + assert part.isidentifier(), f"{part} is not a valid python identifier within {val}" + return val + + +Range: TypeAlias = Union[Tuple[int, int], Tuple[float, float]] +PythonIdentifier: TypeAlias = Annotated[str, AfterValidator(_is_identifier)] +""" +A valid python identifier, including globally namespace pathed like +module.submodule.ClassName +""" +ConfigID: TypeAlias = Annotated[str, Field(pattern=CONFIG_ID_PATTERN)] +""" +A string that refers to a config file by the ``id`` field in that config +""" +ConfigSource: TypeAlias = Union[Path, PathLike[str], ConfigID] +""" +Union of all types of config sources +""" + + +def valid_config_id(val: Any) -> TypeIs[ConfigID]: + """ + Checks whether a string is a valid config id. + """ + return bool(re.fullmatch(CONFIG_ID_PATTERN, val)) + + +class BBox(NamedTuple): + """ + Bounding Box + + (for specificying a rectangular ROI within an image frame) + """ + + x: int + """Leftmost x coordinate""" + y: int + """Topmost y coordinate""" + width: int + height: int + + +class Resolution(NamedTuple): + """ + Pixel resolution of a frame or camera. + + (i.e. the number of pixels a frame is wide and tall, + not e.g. the spatial extent of an individual pixel) + """ + + width: int + height: int \ No newline at end of file diff --git a/miniscope_io/utils.py b/mio/utils.py similarity index 100% rename from miniscope_io/utils.py rename to mio/utils.py diff --git a/miniscope_io/vendor/README.md b/mio/vendor/README.md similarity index 100% rename from miniscope_io/vendor/README.md rename to mio/vendor/README.md diff --git a/miniscope_io/vendor/__init__.py b/mio/vendor/__init__.py similarity index 100% rename from miniscope_io/vendor/__init__.py rename to mio/vendor/__init__.py diff --git a/miniscope_io/vendor/opalkelly/LICENSE b/mio/vendor/opalkelly/LICENSE similarity index 100% rename from miniscope_io/vendor/opalkelly/LICENSE rename to mio/vendor/opalkelly/LICENSE diff --git a/miniscope_io/vendor/opalkelly/README.md b/mio/vendor/opalkelly/README.md similarity index 86% rename from miniscope_io/vendor/opalkelly/README.md rename to mio/vendor/opalkelly/README.md index d27e17ef..a5032c00 100644 --- a/miniscope_io/vendor/opalkelly/README.md +++ b/mio/vendor/opalkelly/README.md @@ -29,7 +29,7 @@ So instead we modify the import location in `_ok.so` like: install_name_tool -change \ @rpath/libokFrontPanel.dylib \ @loader_path/libokFrontPanel.dylib \ - miniscope_io/vendor/opalkelly/mac/_ok.so + mio/vendor/opalkelly/mac/_ok.so ``` ### Linux @@ -38,8 +38,8 @@ Since we can't modify `LD_LIBRARY_PATH` dynamically, we change the location of the loaded `libokFrontPanel.so` to be `$ORIGIN/libokFrontPanel.so` ```bash -patchelf --remove-needed libokFrontPanel.so miniscope_io/vendor/opalkelly/linux/_ok.so -patchelf --add-needed '$ORIGIN/libokFrontPanel.so' miniscope_io/vendor/opalkelly/linux/_ok.so +patchelf --remove-needed libokFrontPanel.so mio/vendor/opalkelly/linux/_ok.so +patchelf --add-needed '$ORIGIN/libokFrontPanel.so' mio/vendor/opalkelly/linux/_ok.so ``` We also need to install `liblua5.3-0` (`apt install liblua5.3-0`)! \ No newline at end of file diff --git a/mio/vendor/opalkelly/__init__.py b/mio/vendor/opalkelly/__init__.py new file mode 100644 index 00000000..244db7f4 --- /dev/null +++ b/mio/vendor/opalkelly/__init__.py @@ -0,0 +1,27 @@ +import sys +from pathlib import Path +import os + + +def patch_env_path(name: str, value: str): + val = os.environ.get(name, None) + if val is None: + val = value + else: + val = ":".join([val.rstrip(":"), value]) + os.environ[name] = val + + +base_path = Path(__file__).parent.resolve() + +if sys.platform == "darwin": + from mio.vendor.opalkelly.mac.ok import * + +elif sys.platform.startswith("linux"): + # Linux + from mio.vendor.opalkelly.linux.ok import * + +elif sys.platform.startswith("win"): + from mio.vendor.opalkelly.win.ok import * +else: + raise ImportError("Dont know what operating system you are on, cant use OpalKelly") diff --git a/miniscope_io/vendor/opalkelly/linux/__init__.py b/mio/vendor/opalkelly/linux/__init__.py similarity index 100% rename from miniscope_io/vendor/opalkelly/linux/__init__.py rename to mio/vendor/opalkelly/linux/__init__.py diff --git a/miniscope_io/vendor/opalkelly/linux/_ok.so b/mio/vendor/opalkelly/linux/_ok.so similarity index 100% rename from miniscope_io/vendor/opalkelly/linux/_ok.so rename to mio/vendor/opalkelly/linux/_ok.so diff --git a/miniscope_io/vendor/opalkelly/linux/libokFrontPanel.so b/mio/vendor/opalkelly/linux/libokFrontPanel.so similarity index 100% rename from miniscope_io/vendor/opalkelly/linux/libokFrontPanel.so rename to mio/vendor/opalkelly/linux/libokFrontPanel.so diff --git a/miniscope_io/vendor/opalkelly/linux/ok.py b/mio/vendor/opalkelly/linux/ok.py similarity index 100% rename from miniscope_io/vendor/opalkelly/linux/ok.py rename to mio/vendor/opalkelly/linux/ok.py diff --git a/miniscope_io/vendor/opalkelly/linux/okFrontPanel.h b/mio/vendor/opalkelly/linux/okFrontPanel.h similarity index 100% rename from miniscope_io/vendor/opalkelly/linux/okFrontPanel.h rename to mio/vendor/opalkelly/linux/okFrontPanel.h diff --git a/miniscope_io/vendor/opalkelly/linux/okFrontPanelDLL.h b/mio/vendor/opalkelly/linux/okFrontPanelDLL.h similarity index 100% rename from miniscope_io/vendor/opalkelly/linux/okFrontPanelDLL.h rename to mio/vendor/opalkelly/linux/okFrontPanelDLL.h diff --git a/miniscope_io/vendor/opalkelly/mac/__init__.py b/mio/vendor/opalkelly/mac/__init__.py similarity index 100% rename from miniscope_io/vendor/opalkelly/mac/__init__.py rename to mio/vendor/opalkelly/mac/__init__.py diff --git a/miniscope_io/vendor/opalkelly/mac/_ok.so b/mio/vendor/opalkelly/mac/_ok.so similarity index 100% rename from miniscope_io/vendor/opalkelly/mac/_ok.so rename to mio/vendor/opalkelly/mac/_ok.so diff --git a/miniscope_io/vendor/opalkelly/mac/libokFrontPanel.dylib b/mio/vendor/opalkelly/mac/libokFrontPanel.dylib similarity index 100% rename from miniscope_io/vendor/opalkelly/mac/libokFrontPanel.dylib rename to mio/vendor/opalkelly/mac/libokFrontPanel.dylib diff --git a/miniscope_io/vendor/opalkelly/mac/ok.py b/mio/vendor/opalkelly/mac/ok.py similarity index 100% rename from miniscope_io/vendor/opalkelly/mac/ok.py rename to mio/vendor/opalkelly/mac/ok.py diff --git a/miniscope_io/vendor/opalkelly/mac/okFrontPanel.h b/mio/vendor/opalkelly/mac/okFrontPanel.h similarity index 100% rename from miniscope_io/vendor/opalkelly/mac/okFrontPanel.h rename to mio/vendor/opalkelly/mac/okFrontPanel.h diff --git a/miniscope_io/vendor/opalkelly/mac/okFrontPanelDLL.h b/mio/vendor/opalkelly/mac/okFrontPanelDLL.h similarity index 100% rename from miniscope_io/vendor/opalkelly/mac/okFrontPanelDLL.h rename to mio/vendor/opalkelly/mac/okFrontPanelDLL.h diff --git a/miniscope_io/vendor/opalkelly/win/__init__.py b/mio/vendor/opalkelly/win/__init__.py similarity index 100% rename from miniscope_io/vendor/opalkelly/win/__init__.py rename to mio/vendor/opalkelly/win/__init__.py diff --git a/miniscope_io/vendor/opalkelly/win/_ok.pyd b/mio/vendor/opalkelly/win/_ok.pyd similarity index 100% rename from miniscope_io/vendor/opalkelly/win/_ok.pyd rename to mio/vendor/opalkelly/win/_ok.pyd diff --git a/miniscope_io/vendor/opalkelly/win/ok.py b/mio/vendor/opalkelly/win/ok.py similarity index 100% rename from miniscope_io/vendor/opalkelly/win/ok.py rename to mio/vendor/opalkelly/win/ok.py diff --git a/miniscope_io/vendor/opalkelly/win/okFrontPanel.dll b/mio/vendor/opalkelly/win/okFrontPanel.dll similarity index 100% rename from miniscope_io/vendor/opalkelly/win/okFrontPanel.dll rename to mio/vendor/opalkelly/win/okFrontPanel.dll diff --git a/miniscope_io/vendor/opalkelly/win/okFrontPanel.h b/mio/vendor/opalkelly/win/okFrontPanel.h similarity index 100% rename from miniscope_io/vendor/opalkelly/win/okFrontPanel.h rename to mio/vendor/opalkelly/win/okFrontPanel.h diff --git a/miniscope_io/vendor/opalkelly/win/okFrontPanel.lib b/mio/vendor/opalkelly/win/okFrontPanel.lib similarity index 100% rename from miniscope_io/vendor/opalkelly/win/okFrontPanel.lib rename to mio/vendor/opalkelly/win/okFrontPanel.lib diff --git a/miniscope_io/vendor/opalkelly/win/okFrontPanelDLL.h b/mio/vendor/opalkelly/win/okFrontPanelDLL.h similarity index 100% rename from miniscope_io/vendor/opalkelly/win/okFrontPanelDLL.h rename to mio/vendor/opalkelly/win/okFrontPanelDLL.h diff --git a/notebooks/Wire-Free-DAQ.ipynb b/notebooks/Wire-Free-DAQ.ipynb index ff96fb3d..11c06957 100644 --- a/notebooks/Wire-Free-DAQ.ipynb +++ b/notebooks/Wire-Free-DAQ.ipynb @@ -156,10 +156,10 @@ "ename": "ValueError", "evalue": "read length must be non-negative or -1", "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mValueError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[0;32m 7\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m5000\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 8\u001b[0m \u001b[0mdataHeader\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfromstring\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m4\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0muint32\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 9\u001b[1;33m \u001b[0mdataHeader\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdataHeader\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfromstring\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdataHeader\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mBUFFER_HEADER_HEADER_LENGTH_POS\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m-\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m*\u001b[0m \u001b[1;36m4\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0muint32\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 10\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 11\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdataHeader\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;31mValueError\u001b[0m: read length must be non-negative or -1" + "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m", + "\u001B[1;31mValueError\u001B[0m Traceback (most recent call last)", + "\u001B[1;32m\u001B[0m in \u001B[0;36m\u001B[1;34m\u001B[0m\n\u001B[0;32m 7\u001B[0m \u001B[1;32mfor\u001B[0m \u001B[0mi\u001B[0m \u001B[1;32min\u001B[0m \u001B[0mrange\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;36m5000\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m 8\u001B[0m \u001B[0mdataHeader\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mnp\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mfromstring\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mf\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mread\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;36m4\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mdtype\u001B[0m\u001B[1;33m=\u001B[0m\u001B[0mnp\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0muint32\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m----> 9\u001B[1;33m \u001B[0mdataHeader\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mnp\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mappend\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mdataHeader\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mnp\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mfromstring\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mf\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mread\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mdataHeader\u001B[0m\u001B[1;33m[\u001B[0m\u001B[0mBUFFER_HEADER_HEADER_LENGTH_POS\u001B[0m\u001B[1;33m]\u001B[0m \u001B[1;33m-\u001B[0m \u001B[1;36m1\u001B[0m\u001B[1;33m)\u001B[0m \u001B[1;33m*\u001B[0m \u001B[1;36m4\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mdtype\u001B[0m\u001B[1;33m=\u001B[0m\u001B[0mnp\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0muint32\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m 10\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m 11\u001B[0m \u001B[0mprint\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mdataHeader\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n", + "\u001B[1;31mValueError\u001B[0m: read length must be non-negative or -1" ] } ], @@ -239,7 +239,7 @@ "outputs": [], "source": [ "# Delete data from SD Card\n", - "f.seek(ataStartSector * sectorSize, 0)\n", + "f.seek(dataStartSector * sectorSize, 0)\n", "\n", "zeros = []\n", "for i in range(512):\n", @@ -249,4 +249,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/notebooks/grab_frames.ipynb b/notebooks/grab_frames.ipynb index e125dde2..27fef4ac 100644 --- a/notebooks/grab_frames.ipynb +++ b/notebooks/grab_frames.ipynb @@ -32,8 +32,8 @@ "import warnings\n", "warnings.filterwarnings(\"ignore\")\n", "\n", - "from miniscope_io.io import SDCard\n", - "from miniscope_io.formats import WireFreeSDLayout" + "from mio.io import SDCard\n", + "from mio.models.sdcard import SDLayout" ] }, { @@ -100,9 +100,7 @@ ] } ], - "source": [ - "pprint(WireFreeSDLayout.dict(), sort_dicts=False)" - ] + "source": "pprint(SDLayout.from_id('wirefree-sd-layout'), sort_dicts=False)" }, { "cell_type": "markdown", @@ -118,7 +116,9 @@ "In the future once I get an image we would want to make some jupyter widget to select possible drives,\n", "but for now i'll just hardcode it as a string for the sake of an example. The `SDCard` class has a\n", "`check_valid` method that looks for the `WRITE_KEY`s as in the prior notebook, so we could also\n", - "make that into a classmethod and just automatically find the right drive that way." + "make that into a classmethod and just automatically find the right drive that way.\n", + "\n", + "Rather than directly referencing the layout object, we can instead just use its `id` field" ] }, { @@ -157,7 +157,7 @@ } ], "source": [ - "sd = SDCard(drive=drive, layout = WireFreeSDLayout)\n", + "sd = SDCard(drive=drive, layout = \"wirefree-sd-layout\")\n", "\n", "pprint(sd.config.dict())" ] @@ -184984,4 +184984,4 @@ }, "nbformat": 4, "nbformat_minor": 1 -} \ No newline at end of file +} diff --git a/notebooks/plot_headers.ipynb b/notebooks/plot_headers.ipynb index ea600839..adef0b64 100644 --- a/notebooks/plot_headers.ipynb +++ b/notebooks/plot_headers.ipynb @@ -22,10 +22,9 @@ "import warnings\n", "warnings.filterwarnings(\"ignore\")\n", "\n", - "from miniscope_io.io import SDCard\n", - "from miniscope_io.formats import WireFreeSDLayout_Battery\n", - "from miniscope_io.models.data import Frames\n", - "from miniscope_io.plots.headers import plot_headers, battery_voltage" + "from mio.io import SDCard\n", + "from mio.models.data import Frames\n", + "from mio.plots.headers import plot_headers, battery_voltage" ], "metadata": { "collapsed": false, @@ -66,7 +65,7 @@ "source": [ "# Recall that you have to use an SDCard layout that matches the data you have!\n", "# Here we are using an updated layout that includes the battery level\n", - "sd = SDCard(drive=drive, layout = WireFreeSDLayout_Battery)" + "sd = SDCard(drive=drive, layout = \"wirefree-sd-layout-battery\")" ], "metadata": { "collapsed": false, diff --git a/notebooks/save_video.ipynb b/notebooks/save_video.ipynb index 7eb08952..e8974957 100644 --- a/notebooks/save_video.ipynb +++ b/notebooks/save_video.ipynb @@ -21,8 +21,7 @@ "outputs": [], "source": [ "from pathlib import Path\n", - "from miniscope_io.io import SDCard\n", - "from miniscope_io.formats import WireFreeSDLayout" + "from mio.io import SDCard" ] }, { @@ -43,7 +42,7 @@ "outputs": [], "source": [ "drive = Path('..') / 'data' / 'wirefree_example.img'\n", - "sd = SDCard(drive=drive, layout = WireFreeSDLayout)" + "sd = SDCard(drive=drive, layout = \"wirefree-sd-layout\")" ], "metadata": { "collapsed": false, @@ -130,4 +129,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} \ No newline at end of file +} diff --git a/pdm.lock b/pdm.lock index 37b10e81..010af761 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "all", "dev", "docs", "plot", "tests"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:cda0e212515e3067b27276916a7033214752eadea01b3513953b8dca9e20cbc9" +content_hash = "sha256:45c1d1f2b3d9add57b72934ccc1ee0e8798c25bd2e094d76b91f9bc2fb61f3e1" [[metadata.targets]] requires_python = "~=3.9" @@ -1361,7 +1361,7 @@ name = "platformdirs" version = "4.3.6" requires_python = ">=3.8" summary = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." -groups = ["all", "dev"] +groups = ["default", "all", "dev"] files = [ {file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"}, {file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"}, @@ -1840,6 +1840,20 @@ files = [ {file = "sphinx_click-6.0.0.tar.gz", hash = "sha256:f5d664321dc0c6622ff019f1e1c84e58ce0cecfddeb510e004cf60c2a3ab465b"}, ] +[[package]] +name = "sphinx-design" +version = "0.6.1" +requires_python = ">=3.9" +summary = "A sphinx extension for designing beautiful, view size responsive web components." +groups = ["all", "docs"] +dependencies = [ + "sphinx<9,>=6", +] +files = [ + {file = "sphinx_design-0.6.1-py3-none-any.whl", hash = "sha256:b11f37db1a802a183d61b159d9a202314d4d2fe29c163437001324fe2f19549c"}, + {file = "sphinx_design-0.6.1.tar.gz", hash = "sha256:b44eea3719386d04d765c1a8257caca2b3e6f8421d7b3a5e742c0fd45f84e632"}, +] + [[package]] name = "sphinxcontrib-applehelp" version = "2.0.0" @@ -1885,18 +1899,17 @@ files = [ ] [[package]] -name = "sphinxcontrib-mermaid" -version = "1.0.0" +name = "sphinxcontrib-programoutput" +version = "0.18" requires_python = ">=3.8" -summary = "Mermaid diagrams in yours Sphinx powered docs" +summary = "Sphinx extension to include program output" groups = ["all", "docs"] dependencies = [ - "pyyaml", - "sphinx", + "Sphinx>=5.0.0", ] files = [ - {file = "sphinxcontrib_mermaid-1.0.0-py3-none-any.whl", hash = "sha256:60b72710ea02087f212028feb09711225fbc2e343a10d34822fe787510e1caa3"}, - {file = "sphinxcontrib_mermaid-1.0.0.tar.gz", hash = "sha256:2e8ab67d3e1e2816663f9347d026a8dee4a858acdd4ad32dd1c808893db88146"}, + {file = "sphinxcontrib_programoutput-0.18-py3-none-any.whl", hash = "sha256:8a651bc85de69a808a064ff0e48d06c12b9347da4fe5fdb1e94914b01e1b0c36"}, + {file = "sphinxcontrib_programoutput-0.18.tar.gz", hash = "sha256:09e68b6411d937a80b6085f4fdeaa42e0dc5555480385938465f410589d2eed8"}, ] [[package]] @@ -1933,6 +1946,17 @@ files = [ {file = "tomli-2.0.2.tar.gz", hash = "sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed"}, ] +[[package]] +name = "tomli-w" +version = "1.1.0" +requires_python = ">=3.9" +summary = "A lil' TOML writer" +groups = ["all", "tests"] +files = [ + {file = "tomli_w-1.1.0-py3-none-any.whl", hash = "sha256:1403179c78193e3184bfaade390ddbd071cba48a32a2e62ba11aae47490c63f7"}, + {file = "tomli_w-1.1.0.tar.gz", hash = "sha256:49e847a3a304d516a169a601184932ef0f6b61623fe680f836a2aa7128ed0d33"}, +] + [[package]] name = "tqdm" version = "4.66.6" diff --git a/pyproject.toml b/pyproject.toml index a045c4a1..c619d1d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,13 @@ [project] -name = "miniscope_io" -version = "0.4.1" +name = "mio" description = "Generic I/O for miniscopes" authors = [ {name = "sneakers-the-rat", email = "sneakers-the-rat@protonmail.com"}, {name = "t-sasatani", email = "sasatani.dev@gmail.com"}, ] license = {text = "AGPL-3.0"} +dynamic = ["version"] + requires-python = "<4.0,>=3.9" dependencies = [ "opencv-python>=4.7.0.72", @@ -20,7 +21,8 @@ dependencies = [ "rich>=13.6.0", "pyyaml>=6.0.1", "click>=8.1.7", - "typing-extensions>=4.12.2; python_version<'3.11'", + "platformdirs>=4.3.6", + 'typing-extensions>=4.12.2; python_version<"3.13"', ] readme = "README.md" @@ -50,7 +52,7 @@ classifiers = [ [project.urls] homepage = "https://miniscope-io.readthedocs.io/" -repository = "https://github.com/Aharoni-Lab/miniscope-io" +repository = "https://github.com/Aharoni-Lab/mio" documentation = "https://miniscope-io.readthedocs.io/" [project.optional-dependencies] @@ -59,7 +61,8 @@ tests = [ "pytest>=8.2.2", "pytest-cov>=5.0.0", "pytest-timeout>=2.3.1", - "miniscope_io[plot]", + "mio[plot]", + "tomli-w>=1.1.0", ] docs = [ "sphinx>=6.2.1", @@ -67,6 +70,8 @@ docs = [ "furo>2023.07.26", "myst-parser>3.0.0", "autodoc-pydantic>=2.0.1", + "sphinxcontrib-programoutput>=0.17", + "sphinx-design>=0.6.1", "sphinxcontrib-mermaid>=1.0.0", ] dev = [ @@ -75,25 +80,30 @@ dev = [ "pre-commit>=3.7.1", ] all = [ - "miniscope_io[tests,docs,dev]" + "mio[tests,docs,dev]" ] [project.scripts] -mio = "miniscope_io.cli.main:cli" +mio = "mio.cli.main:cli" [tool.pdm.scripts] test = "pytest" lint.composite = [ "ruff check", - "black miniscope_io --diff" + "black mio --diff" ] format.composite = [ "ruff check --fix", - "black miniscope_io" + "black mio" ] [tool.pdm.build] -includes = ["miniscope_io"] +includes = ["mio"] + +[tool.pdm.version] +source = "scm" +tag_filter = "v*" +tag_regex = 'v(?P([1-9][0-9]*!)?(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))*((a|b|c|rc)(0|[1-9][0-9]*))?(\.post(0|[1-9][0-9]*))?(\.dev(0|[1-9][0-9]*))?$)$' [build-system] requires = ["pdm-backend"] @@ -101,7 +111,7 @@ build-backend = "pdm.backend" [tool.pytest.ini_options] addopts = [ - "--cov=miniscope_io", + "--cov=mio", "--cov-append", ] filterwarnings = [ @@ -115,14 +125,14 @@ timeout = 60 [tool.coverage.run] omit = [ - "miniscope_io/vendor/*", - "miniscope_io/sources/opalkelly.py", # can't test hardware interface directly + "mio/vendor/*", + "mio/sources/opalkelly.py", # can't test hardware interface directly ] [tool.ruff] target-version = "py39" -include = ["miniscope_io/**/*.py", "pyproject.toml"] -exclude = ["docs", "tests", "miniscope_io/vendor", "noxfile.py"] +include = ["mio/**/*.py", "pyproject.toml"] +exclude = ["docs", "tests", "mio/vendor", "noxfile.py"] line-length = 100 [tool.ruff.lint] @@ -153,6 +163,8 @@ select = [ "D210", "D211", # emptiness "D419", + # snake case function name + "N802" ] ignore = [ "ANN101", "ANN102", "ANN401", "ANN204", @@ -173,7 +185,7 @@ plugins = [ "pydantic.mypy" ] packages = [ - "miniscope_io" + "mio" ] exclude = [ '.*vendor.*' @@ -183,6 +195,6 @@ warn_unreachable = true [tool.black] target-version = ['py38', 'py39', 'py310', 'py311'] -include = "miniscope_io/.*\\.py$" -extend-exclude = 'miniscope_io/vendor/.*' +include = "mio/.*\\.py$" +extend-exclude = 'mio/vendor/.*' line-length = 100 diff --git a/tests/conftest.py b/tests/conftest.py index 59e7cbd1..8d8c2900 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,6 +6,10 @@ import pytest import yaml +from mio.models.mixins import ConfigYAMLMixin + +from .fixtures import * + DATA_DIR = Path(__file__).parent / "data" CONFIG_DIR = DATA_DIR / "config" MOCK_DIR = Path(__file__).parent / "mock" @@ -25,14 +29,29 @@ def pytest_sessionstart(session): @pytest.fixture(autouse=True) def mock_okdev(monkeypatch): - from miniscope_io.sources.mocks import okDevMock - from miniscope_io.sources import opalkelly - from miniscope_io import stream_daq + from mio.sources.mocks import okDevMock + from mio.sources import opalkelly + from mio import stream_daq monkeypatch.setattr(opalkelly, "okDev", okDevMock) monkeypatch.setattr(stream_daq, "okDev", okDevMock) +@pytest.fixture(scope="session", autouse=True) +def mock_config_source(monkeypatch_session): + """ + Add the `tests/data/config` directory to the config sources for the entire testing session + """ + current_sources = ConfigYAMLMixin.config_sources + + @classmethod + @property + def _config_sources(cls: type[ConfigYAMLMixin]) -> list[Path]: + return [CONFIG_DIR, *current_sources] + + monkeypatch_session.setattr(ConfigYAMLMixin, "config_sources", _config_sources) + + @pytest.fixture() def set_okdev_input(monkeypatch): """ @@ -41,7 +60,7 @@ def set_okdev_input(monkeypatch): """ def _set_okdev_input(file: Union[str, Path]): - from miniscope_io.sources.mocks import okDevMock + from mio.sources.mocks import okDevMock monkeypatch.setattr(okDevMock, "DATA_FILE", file) os.environ["PYTEST_OKDEV_DATA_FILE"] = str(file) diff --git a/tests/data/config/preamble_hex.yml b/tests/data/config/preamble_hex.yml index 4fce97f0..9da2e019 100644 --- a/tests/data/config/preamble_hex.yml +++ b/tests/data/config/preamble_hex.yml @@ -1,3 +1,7 @@ +id: test-wireless-preamble-hex +mio_model: mio.models.stream.StreamDevConfig +mio_version: "v5.0.0" + # capture device. "OK" (Opal Kelly) or "UART" device: "OK" diff --git a/tests/data/config/stream_daq_test_200px.yml b/tests/data/config/stream_daq_test_200px.yml index 2e9fe911..757c9c97 100644 --- a/tests/data/config/stream_daq_test_200px.yml +++ b/tests/data/config/stream_daq_test_200px.yml @@ -1,3 +1,7 @@ +id: test-wireless-200px +mio_model: mio.models.stream.StreamDevConfig +mio_version: "v5.0.0" + # capture device. "OK" (Opal Kelly) or "UART" device: "OK" diff --git a/tests/data/config/wireless_example.yml b/tests/data/config/wireless_example.yml index 42496165..5b3e84f5 100644 --- a/tests/data/config/wireless_example.yml +++ b/tests/data/config/wireless_example.yml @@ -1,3 +1,7 @@ +id: test-wireless-example +mio_model: mio.models.stream.StreamDevConfig +mio_version: "v5.0.0" + # capture device. "OK" (Opal Kelly) or "UART" device: "OK" diff --git a/tests/fixtures.py b/tests/fixtures.py index 508d2d3b..1ae7ef53 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,10 +1,16 @@ from pathlib import Path +from typing import Callable, Optional, Any, MutableMapping import pytest +import yaml +import tomli_w +from _pytest.monkeypatch import MonkeyPatch -from miniscope_io import SDCard -from miniscope_io.formats import WireFreeSDLayout, WireFreeSDLayout_Battery -from miniscope_io.models.data import Frames +from mio import Config +from mio.io import SDCard +from mio.models.config import _global_config_path, set_user_dir +from mio.models.data import Frames +from mio.models.mixins import ConfigYAMLMixin, YamlDumper @pytest.fixture @@ -14,14 +20,14 @@ def wirefree() -> SDCard: """ sd_path = Path(__file__).parent.parent / "data" / "wirefree_example.img" - sdcard = SDCard(drive=sd_path, layout=WireFreeSDLayout) + sdcard = SDCard(drive=sd_path, layout="wirefree-sd-layout") return sdcard @pytest.fixture def wirefree_battery() -> SDCard: sd_path = Path(__file__).parent.parent / "data" / "wirefree_battery_sample.img" - sdcard = SDCard(drive=sd_path, layout=WireFreeSDLayout_Battery) + sdcard = SDCard(drive=sd_path, layout="wirefree-sd-layout-battery") return sdcard @@ -36,3 +42,234 @@ def wirefree_frames(wirefree) -> Frames: except StopIteration: break return Frames(frames=frames) + + +@pytest.fixture() +def tmp_config_source(tmp_path, monkeypatch) -> Path: + """ + Monkeypatch the config sources to include a temporary path + """ + + path = tmp_path / "configs" + path.mkdir(exist_ok=True) + current_sources = ConfigYAMLMixin.config_sources + + @classmethod + @property + def _config_sources(cls: type[ConfigYAMLMixin]) -> list[Path]: + return [path, *current_sources] + + monkeypatch.setattr(ConfigYAMLMixin, "config_sources", _config_sources) + return path + + +@pytest.fixture() +def yaml_config( + tmp_config_source, tmp_path, monkeypatch +) -> Callable[[str, dict, Optional[Path]], Path]: + out_file = tmp_config_source / "test_config.yaml" + + def _yaml_config(id: str, data: dict, path: Optional[Path] = None) -> Path: + if path is None: + path = out_file + else: + path = Path(path) + if not path.is_absolute(): + # put under tmp_path (rather than tmp_config_source) + # in case putting a file outside the config dir is intentional. + path = tmp_path / path + + if path.is_dir(): + path.mkdir(exist_ok=True, parents=True) + path = path / "test_config.yaml" + else: + path.parent.mkdir(exist_ok=True, parents=True) + + data = {"id": id, **data} + with open(path, "w") as yfile: + yaml.dump(data, yfile) + return path + + return _yaml_config + + +@pytest.fixture(scope="session") +def monkeypatch_session() -> MonkeyPatch: + """ + Monkeypatch you can use at the session scope! + """ + mpatch = MonkeyPatch() + yield mpatch + mpatch.undo() + + +@pytest.fixture(scope="session", autouse=True) +def dodge_existing_configs(tmp_path_factory): + """ + Suspend any existing global config file during config tests + """ + tmp_path = tmp_path_factory.mktemp("config_backup") + global_config_path = _global_config_path + backup_global_config_path = tmp_path / "mio_config.yaml.global.bak" + + user_config_path = list(Config().user_dir.glob("mio_config.*")) + if len(user_config_path) == 0: + user_config_path = None + else: + user_config_path = user_config_path[0] + + backup_user_config_path = tmp_path / "mio_config.yaml.user.bak" + + dotenv_path = Path(".env").resolve() + dotenv_backup_path = tmp_path / "dotenv.bak" + + if global_config_path.exists(): + global_config_path.rename(backup_global_config_path) + if user_config_path is not None and user_config_path.exists(): + user_config_path.rename(backup_user_config_path) + if dotenv_path.exists(): + dotenv_path.rename(dotenv_backup_path) + + yield + + if backup_global_config_path.exists(): + global_config_path.unlink(missing_ok=True) + backup_global_config_path.rename(global_config_path) + if backup_user_config_path.exists(): + user_config_path.unlink(missing_ok=True) + backup_user_config_path.rename(user_config_path) + if dotenv_backup_path.exists(): + dotenv_path.unlink(missing_ok=True) + dotenv_backup_path.rename(dotenv_path) + + +@pytest.fixture() +def tmp_cwd(tmp_path, monkeypatch) -> Path: + monkeypatch.chdir(tmp_path) + return tmp_path + + +@pytest.fixture() +def set_env(monkeypatch) -> Callable[[dict[str, Any]], None]: + """ + Function fixture to set environment variables using a nested dict + matching a GlobalConfig.model_dump() + """ + + def _set_env(config: dict[str, Any]) -> None: + for key, value in _flatten(config).items(): + key = "MIO_" + key.upper() + monkeypatch.setenv(key, str(value)) + + return _set_env + + +@pytest.fixture() +def set_dotenv(tmp_cwd) -> Callable[[dict[str, Any]], Path]: + """ + Function fixture to set config variables in a .env file + """ + dotenv_path = tmp_cwd / ".env" + + def _set_dotenv(config: dict[str, Any]) -> Path: + with open(dotenv_path, "w") as dfile: + for key, value in _flatten(config).items(): + key = "MIO_" + key.upper() + dfile.write(f"{key}={value}\n") + return dotenv_path + + return _set_dotenv + + +@pytest.fixture() +def set_pyproject(tmp_cwd) -> Callable[[dict[str, Any]], Path]: + """ + Function fixture to set config variables in a pyproject.toml file + """ + toml_path = tmp_cwd / "pyproject.toml" + + def _set_pyproject(config: dict[str, Any]) -> Path: + config = {"tool": {"mio": {"config": config}}} + + with open(toml_path, "wb") as tfile: + tomli_w.dump(config, tfile) + + return toml_path + + return _set_pyproject + + +@pytest.fixture() +def set_local_yaml(tmp_cwd) -> Callable[[dict[str, Any]], Path]: + """ + Function fixture to set config variables in a mio_config.yaml file in the current directory + """ + yaml_path = tmp_cwd / "mio_config.yaml" + + def _set_local_yaml(config: dict[str, Any]) -> Path: + with open(yaml_path, "w") as yfile: + yaml.dump(config, yfile, Dumper=YamlDumper) + return yaml_path + + return _set_local_yaml + + +@pytest.fixture() +def set_user_yaml(tmp_path) -> Callable[[dict[str, Any]], Path]: + """ + Function fixture to set config variables in a user config file + """ + yaml_path = tmp_path / "user" / "mio_config.yaml" + yaml_path.parent.mkdir(exist_ok=True) + + def _set_user_yaml(config: dict[str, Any]) -> Path: + with open(yaml_path, "w") as yfile: + yaml.dump(config, yfile, Dumper=YamlDumper) + set_user_dir(yaml_path.parent) + return yaml_path + + yield _set_user_yaml + + _global_config_path.unlink(missing_ok=True) + + +@pytest.fixture() +def set_global_yaml() -> Callable[[dict[str, Any]], Path]: + """ + Function fixture to reversibly set config variables in a global mio_config.yaml file + """ + + def _set_global_yaml(config: dict[str, Any]) -> Path: + with open(_global_config_path, "w") as gfile: + yaml.dump(config, gfile, Dumper=YamlDumper) + return _global_config_path + + yield _set_global_yaml + + _global_config_path.unlink(missing_ok=True) + + +@pytest.fixture( + params=[ + "set_env", + "set_dotenv", + "set_pyproject", + "set_local_yaml", + "set_user_yaml", + "set_global_yaml", + ] +) +def set_config(request) -> Callable[[dict[str, Any]], Path]: + return request.getfixturevalue(request.param) + + +def _flatten(d, parent_key="", separator="__") -> dict: + """https://stackoverflow.com/a/6027615/13113166""" + items = [] + for key, value in d.items(): + new_key = parent_key + separator + key if parent_key else key + if isinstance(value, MutableMapping): + items.extend(_flatten(value, new_key, separator=separator).items()) + else: + items.append((new_key, value)) + return dict(items) diff --git a/tests/test_bit_operation.py b/tests/test_bit_operation.py index a37f59d6..6f828c37 100644 --- a/tests/test_bit_operation.py +++ b/tests/test_bit_operation.py @@ -1,29 +1,41 @@ import pytest import numpy as np -from miniscope_io.bit_operation import BufferFormatter +from mio.bit_operation import BufferFormatter -@pytest.mark.parametrize("test_input,header_length_words,preamble_length_words,reverse_header_bits,reverse_header_bytes,reverse_payload_bits,reverse_payload_bytes,expected_header,expected_payload", + +@pytest.mark.parametrize( + "test_input,header_length_words,preamble_length_words,reverse_header_bits,reverse_header_bytes,reverse_payload_bits,reverse_payload_bytes,expected_header,expected_payload", [ - (b'\x12\x34\x56\x78\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xAA\xBB', - 1, - 0, - False, - False, - False, - False, - np.array([0x78563412], dtype=np.uint32), - np.array([0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xAA, 0xBB], dtype=np.uint8)), - (b'\x12\x34\x56\x78\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xAA\xBB', - 1, - 0, - True, - True, - True, - True, - np.array([0x1E6A2C48], dtype=np.uint32), - np.array([0x00, 0x88, 0x44, 0xCC, 0x22, 0xAA, 0x66, 0xEE, 0x11, 0x99, 0x55, 0xDD], dtype=np.uint8)), + ( + b"\x12\x34\x56\x78\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xAA\xBB", + 1, + 0, + False, + False, + False, + False, + np.array([0x78563412], dtype=np.uint32), + np.array( + [0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xAA, 0xBB], + dtype=np.uint8, + ), + ), + ( + b"\x12\x34\x56\x78\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xAA\xBB", + 1, + 0, + True, + True, + True, + True, + np.array([0x1E6A2C48], dtype=np.uint32), + np.array( + [0x00, 0x88, 0x44, 0xCC, 0x22, 0xAA, 0x66, 0xEE, 0x11, 0x99, 0x55, 0xDD], + dtype=np.uint8, + ), + ), ], - ) +) def test_bytebuffer_to_ndarrays( test_input, header_length_words, @@ -33,8 +45,17 @@ def test_bytebuffer_to_ndarrays( reverse_payload_bits, reverse_payload_bytes, expected_header, - expected_payload): - header, payload = BufferFormatter.bytebuffer_to_ndarrays(test_input, header_length_words, preamble_length_words, reverse_header_bits, reverse_header_bytes, reverse_payload_bits, reverse_payload_bytes) + expected_payload, +): + header, payload = BufferFormatter.bytebuffer_to_ndarrays( + test_input, + header_length_words, + preamble_length_words, + reverse_header_bits, + reverse_header_bytes, + reverse_payload_bits, + reverse_payload_bytes, + ) """ Test for ensuring that the conversion and optional bit/byte reversals are performed correctly. The buffer is a concat of a 32-bit array for metadata and an 8-bit array for payload. @@ -43,32 +64,43 @@ def test_bytebuffer_to_ndarrays( np.testing.assert_array_equal(header, expected_header) np.testing.assert_array_equal(payload, expected_payload) -@pytest.mark.parametrize("input_array,expected_output", [ - (np.array([0b11000011101010000100000000000000], dtype=np.uint32), - np.array([0b00000000000000100001010111000011], dtype=np.uint32)), - (np.array([0b00000000000001111000000000000111], dtype=np.uint32), - np.array([0b11100000000000011110000000000000], dtype=np.uint32)), - (np.array([0b10101010101010101010101010101010], dtype=np.uint32), - np.array([0b01010101010101010101010101010101], dtype=np.uint32)), -]) + +@pytest.mark.parametrize( + "input_array,expected_output", + [ + ( + np.array([0b11000011101010000100000000000000], dtype=np.uint32), + np.array([0b00000000000000100001010111000011], dtype=np.uint32), + ), + ( + np.array([0b00000000000001111000000000000111], dtype=np.uint32), + np.array([0b11100000000000011110000000000000], dtype=np.uint32), + ), + ( + np.array([0b10101010101010101010101010101010], dtype=np.uint32), + np.array([0b01010101010101010101010101010101], dtype=np.uint32), + ), + ], +) def test_reverse_bits_in_array(input_array, expected_output): """ Test for flipping bit order in a 32-bit word. """ result = BufferFormatter._reverse_bits_in_array(input_array) np.testing.assert_array_equal(result, expected_output) - -@pytest.mark.parametrize("input_array,expected_output", [ - (np.array([0x12345678], dtype=np.uint32), - np.array([0x78563412], dtype=np.uint32)), - (np.array([0xABCD4574], dtype=np.uint32), - np.array([0x7445CDAB], dtype=np.uint32)), - (np.array([0x11002200], dtype=np.uint32), - np.array([0x00220011], dtype=np.uint32)), -]) + + +@pytest.mark.parametrize( + "input_array,expected_output", + [ + (np.array([0x12345678], dtype=np.uint32), np.array([0x78563412], dtype=np.uint32)), + (np.array([0xABCD4574], dtype=np.uint32), np.array([0x7445CDAB], dtype=np.uint32)), + (np.array([0x11002200], dtype=np.uint32), np.array([0x00220011], dtype=np.uint32)), + ], +) def test_reverse_byte_order_in_array(input_array, expected_output): """ Test for flipping byte order (8-bit chunks) in a 32-bit word. """ result = BufferFormatter._reverse_byte_order_in_array(input_array) - np.testing.assert_array_equal(result, expected_output) \ No newline at end of file + np.testing.assert_array_equal(result, expected_output) diff --git a/tests/test_cli.py b/tests/test_cli.py index 331c7a7d..62fd9eed 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,8 +1,97 @@ +import pdb + import pytest from click.testing import CliRunner +from mio.cli.config import config +from mio import Config +from mio.models import config as _config_mod + @pytest.mark.skip("Needs to be implemented") def test_cli_stream(): """should be able to invoke streamdaq, using various capture options""" pass + + +def test_cli_config_show(): + """ + `mio config` should show current config + """ + runner = CliRunner() + result = runner.invoke(config) + cfg_yaml = Config().to_yaml() + assert cfg_yaml in result.output + + +def test_cli_config_show_global(): + """ + `mio config global` should show contents of the global config file + """ + runner = CliRunner() + result = runner.invoke(config, ["global"]) + cfg_yaml = _config_mod._global_config_path.read_text() + assert str(_config_mod._global_config_path) in result.output + assert cfg_yaml in result.output + + +def test_cli_config_global_path(): + """ + `mio global path` should show the path to the global config file + """ + runner = CliRunner() + result = runner.invoke(config, ["global", "path"]) + assert str(_config_mod._global_config_path) in result.output + + +def test_cli_config_user_show(set_user_yaml): + """ + `mio config user` should show contents of the user config file + """ + user_yaml_path = set_user_yaml({"logs": {"level": "WARNING"}}) + runner = CliRunner() + result = runner.invoke(config, ["user"]) + user_config = user_yaml_path.read_text() + assert "level: WARNING" in user_config + assert user_config in result.output + + +@pytest.mark.parametrize("clean", [True, False]) +@pytest.mark.parametrize("dry_run", [True, False]) +def test_cli_config_user_create(clean, dry_run, tmp_path): + """ + `mio config user create` creates a new user config file, + optionally with clean/dirty mode or dry_run or not + """ + dry_run_cmd = "--dry-run" if dry_run else "--no-dry-run" + clean_cmd = "--clean" if clean else "--dirty" + + config_path = tmp_path / "mio_config.yaml" + + runner = CliRunner() + result = runner.invoke(config, ["user", "create", dry_run_cmd, clean_cmd, str(config_path)]) + + if dry_run: + assert "DRY RUN" in result.output + assert not config_path.exists() + else: + assert "DRY RUN" not in result.output + assert config_path.exists() + + if clean: + assert "level" not in result.output + else: + assert "level" in result.output + + assert f"user_dir: {str(config_path.parent)}" in result.output + + +def test_cli_config_user_path(set_env, set_user_yaml): + """ + `mio config user path` should show the path to the user config file + """ + user_config_path = set_user_yaml({"logs": {"level": "WARNING"}}) + + runner = CliRunner() + result = runner.invoke(config, ["user", "path"]) + assert str(user_config_path) in result.output diff --git a/tests/test_config.py b/tests/test_config.py index 3bd488d4..585abe81 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,35 +1,62 @@ import os from pathlib import Path -from miniscope_io import Config + +import yaml +import numpy as np + +from mio import Config +from mio.models.config import _global_config_path, set_user_dir +from tests.fixtures import ( + set_env, + set_dotenv, + set_pyproject, + set_local_yaml, + set_user_yaml, + set_global_yaml, + set_config, +) + def test_config(tmp_path): """ Config should be able to make directories and set sensible defaults """ - config = Config(base_dir = tmp_path) - assert config.base_dir.exists() + config = Config(user_dir=tmp_path) + assert config.user_dir.exists() assert config.log_dir.exists() - assert config.log_dir == config.base_dir / 'logs' + assert config.log_dir == config.user_dir / "logs" + + +def test_set_config(set_config, tmp_path): + """We should be able to set parameters from all available modalities""" + file_n = int(np.random.default_rng().integers(0, 100)) + user_dir = tmp_path / f"fake/dir/{np.random.default_rng().integers(0, 100)}" + + set_config({"user_dir": str(user_dir), "logs": {"file_n": file_n}}) + + config = Config() + assert config.user_dir == user_dir + assert config.logs.file_n == file_n def test_config_from_environment(tmp_path): """ Setting environmental variables should set the config, including recursive models """ - os.environ['MINISCOPE_IO_BASE_DIR'] = str(tmp_path) + os.environ["MIO_USER_DIR"] = str(tmp_path) # we can also override the default log dir name - override_logdir = Path(tmp_path) / 'fancylogdir' - os.environ['MINISCOPE_IO_LOG_DIR'] = str(override_logdir) + override_logdir = Path(tmp_path) / "fancylogdir" + os.environ["MIO_LOG_DIR"] = str(override_logdir) # and also recursive models - os.environ['MINISCOPE_IO_LOGS__LEVEL'] = 'error' + os.environ["MIO_LOGS__LEVEL"] = "error" config = Config() - assert config.base_dir == Path(tmp_path) + assert config.user_dir == Path(tmp_path) assert config.log_dir == override_logdir - assert config.logs.level == 'error'.upper() - del os.environ['MINISCOPE_IO_BASE_DIR'] - del os.environ['MINISCOPE_IO_LOG_DIR'] - del os.environ['MINISCOPE_IO_LOGS__LEVEL'] + assert config.logs.level == "error".upper() + del os.environ["MIO_USER_DIR"] + del os.environ["MIO_LOG_DIR"] + del os.environ["MIO_LOGS__LEVEL"] def test_config_from_dotenv(tmp_path): @@ -38,10 +65,52 @@ def test_config_from_dotenv(tmp_path): this test can be more relaxed since its basically a repetition of previous """ - tmp_path.mkdir(exist_ok=True,parents=True) - dotenv = tmp_path / '.env' - with open(dotenv, 'w') as denvfile: - denvfile.write(f'MINISCOPE_IO_BASE_DIR={str(tmp_path)}') + tmp_path.mkdir(exist_ok=True, parents=True) + dotenv = tmp_path / ".env" + with open(dotenv, "w") as denvfile: + denvfile.write(f"MIO_USER_DIR={str(tmp_path)}") + + config = Config(_env_file=dotenv, _env_file_encoding="utf-8") + assert config.user_dir == Path(tmp_path) + + +def test_set_user_dir(tmp_path): + """ + We should be able to set the user dir and the global config should respect it + """ + user_config = tmp_path / "mio_config.yml" + file_n = int(np.random.default_rng().integers(0, 100)) + with open(user_config, "w") as yfile: + yaml.dump({"logs": {"file_n": file_n}}, yfile) + + set_user_dir(tmp_path) + + with open(_global_config_path, "r") as gfile: + global_config = yaml.safe_load(gfile) + + assert global_config["user_dir"] == str(tmp_path) + assert Config().user_dir == tmp_path + assert Config().logs.file_n == file_n + + # we do this manual cleanup here and not in a fixture because we are testing + # that the things we are doing in the fixtures are working correctly! + _global_config_path.unlink(missing_ok=True) + - config = Config(_env_file=dotenv, _env_file_encoding='utf-8') - assert config.base_dir == Path(tmp_path) +def test_config_sources_overrides( + set_env, set_dotenv, set_pyproject, set_local_yaml, set_user_yaml, set_global_yaml +): + """Test that the different config sources are overridden in the correct order""" + set_global_yaml({"logs": {"file_n": 0}}) + assert Config().logs.file_n == 0 + set_user_yaml({"logs": {"file_n": 1}}) + assert Config().logs.file_n == 1 + set_pyproject({"logs": {"file_n": 2}}) + assert Config().logs.file_n == 2 + set_local_yaml({"logs": {"file_n": 3}}) + assert Config().logs.file_n == 3 + set_dotenv({"logs": {"file_n": 4}}) + assert Config().logs.file_n == 4 + set_env({"logs": {"file_n": 5}}) + assert Config().logs.file_n == 5 + assert Config(**{"logs": {"file_n": 6}}).logs.file_n == 6 diff --git a/tests/test_data.py b/tests/test_data.py index e4191a05..d69d52d7 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -3,7 +3,7 @@ import pytest from .fixtures import wirefree_frames, wirefree import pandas as pd -from miniscope_io.models.sdcard import SDBufferHeader +from mio.models.sdcard import SDBufferHeader @pytest.mark.filterwarnings("ignore:Pydantic serializer warnings") diff --git a/tests/test_device_update.py b/tests/test_device_update.py index ca1da6d2..6dae61ed 100644 --- a/tests/test_device_update.py +++ b/tests/test_device_update.py @@ -2,77 +2,101 @@ import serial from pydantic import ValidationError from unittest.mock import MagicMock, patch, call -from miniscope_io.models.devupdate import UpdateCommandDefinitions, UpdateTarget -from miniscope_io.device_update import device_update, find_ftdi_device +from mio.models.devupdate import UpdateCommandDefinitions, UpdateKey +from mio.device_update import device_update, find_ftdi_device + @pytest.fixture def mock_serial_fixture(request): device_list = request.param - with patch('serial.Serial') as mock_serial, patch('serial.tools.list_ports.comports') as mock_comports: + with patch("serial.Serial") as mock_serial, patch( + "serial.tools.list_ports.comports" + ) as mock_comports: mock_serial_instance = mock_serial.return_value - mock_comports.return_value = [MagicMock(vid=device['vid'], pid=device['pid'], device=device['device']) - for device in device_list] + mock_comports.return_value = [ + MagicMock(vid=device["vid"], pid=device["pid"], device=device["device"]) + for device in device_list + ] yield mock_serial, mock_comports, mock_serial_instance -@pytest.mark.parametrize('mock_serial_fixture', [ - [{'vid': 0x0403, 'pid': 0x6001, 'device': 'COM3'}, - {'vid': 0x0111, 'pid': 0x6111, 'device': 'COM2'}], -], indirect=True) + +@pytest.mark.parametrize( + "mock_serial_fixture", + [ + [ + {"vid": 0x0403, "pid": 0x6001, "device": "COM3"}, + {"vid": 0x0111, "pid": 0x6111, "device": "COM2"}, + ], + ], + indirect=True, +) def test_devupdate_with_device_connected(mock_serial_fixture): """ Test device_update function with a device connected. """ mock_serial, mock_comports, mock_serial_instance = mock_serial_fixture - target = "LED" + key = "LED" value = 2 device_id = 1 port = "COM3" - device_update(target, value, device_id, port) + device_update(key, value, device_id, port) mock_serial.assert_called_once_with(port=port, baudrate=2400, timeout=5, stopbits=2) id_command = (device_id + UpdateCommandDefinitions.id_header) & 0xFF - target_command = (UpdateTarget.LED.value + UpdateCommandDefinitions.target_header) & 0xFF + key_command = (UpdateKey.LED.value + UpdateCommandDefinitions.key_header) & 0xFF value_LSB_command = ( (value & UpdateCommandDefinitions.LSB_value_mask) + UpdateCommandDefinitions.LSB_header ) & 0xFF value_MSB_command = ( - ((value & UpdateCommandDefinitions.MSB_value_mask) >> 6) + UpdateCommandDefinitions.MSB_header + ((value & UpdateCommandDefinitions.MSB_value_mask) >> 6) + + UpdateCommandDefinitions.MSB_header ) & 0xFF reset_command = UpdateCommandDefinitions.reset_byte expected_calls = [ - call(id_command.to_bytes(1, 'big')), - call(target_command.to_bytes(1, 'big')), - call(value_LSB_command.to_bytes(1, 'big')), - call(value_MSB_command.to_bytes(1, 'big')), - call(reset_command.to_bytes(1, 'big')), + call(id_command.to_bytes(1, "big")), + call(key_command.to_bytes(1, "big")), + call(value_LSB_command.to_bytes(1, "big")), + call(value_MSB_command.to_bytes(1, "big")), + call(reset_command.to_bytes(1, "big")), ] assert mock_serial_instance.write.call_count == len(expected_calls) mock_serial_instance.write.assert_has_calls(expected_calls, any_order=False) -@pytest.mark.parametrize('mock_serial_fixture', [ - [], -], indirect=True) +@pytest.mark.parametrize( + "mock_serial_fixture", + [ + [], + ], + indirect=True, +) def test_devupdate_without_device_connected(mock_serial_fixture): """ Test device_update function without a device connected. """ - target = "GAIN" + key = "GAIN" value = 2 device_id = 0 with pytest.raises(ValueError, match="No FTDI devices found."): - device_update(target, value, device_id) - -@pytest.mark.parametrize('mock_serial_fixture', [ - [{'vid': 0x0403, 'pid': 0x6001, 'device': 'COM3'}, - {'vid': 0x0111, 'pid': 0x6111, 'device': 'COM2'}], -], indirect=True) + device_update(key, value, device_id) + + +@pytest.mark.parametrize( + "mock_serial_fixture", + [ + [ + {"vid": 0x0403, "pid": 0x6001, "device": "COM3"}, + {"vid": 0x0111, "pid": 0x6111, "device": "COM2"}, + ], + ], + indirect=True, +) def test_find_ftdi_device(mock_serial_fixture): """ Test find_ftdi_device function. @@ -81,60 +105,86 @@ def test_find_ftdi_device(mock_serial_fixture): assert result == ["COM3"] -@pytest.mark.parametrize('mock_serial_fixture', [ - [{'vid': 0x0403, 'pid': 0x6001, 'device': 'COM3'}, - {'vid': 0x0111, 'pid': 0x6111, 'device': 'COM2'}], -], indirect=True) -def test_invalid_target_raises_error(mock_serial_fixture): + +@pytest.mark.parametrize( + "mock_serial_fixture", + [ + [ + {"vid": 0x0403, "pid": 0x6001, "device": "COM3"}, + {"vid": 0x0111, "pid": 0x6111, "device": "COM2"}, + ], + ], + indirect=True, +) +def test_invalid_key_raises_error(mock_serial_fixture): """ - Test that an invalid target raises an error. + Test that an invalid key raises an error. """ - target = "RANDOM_STRING" + key = "RANDOM_STRING" value = 50 device_id = 1 port = "COM3" - with pytest.raises(ValidationError, match="Target RANDOM_STRING not found"): - device_update(target, value, device_id, port) + with pytest.raises(ValidationError, match="Key RANDOM_STRING not found"): + device_update(key, value, device_id, port) -@pytest.mark.parametrize('mock_serial_fixture', [ - [{'vid': 0x0403, 'pid': 0x6001, 'device': 'COM3'}, - {'vid': 0x0111, 'pid': 0x6111, 'device': 'COM2'}], -], indirect=True) + +@pytest.mark.parametrize( + "mock_serial_fixture", + [ + [ + {"vid": 0x0403, "pid": 0x6001, "device": "COM3"}, + {"vid": 0x0111, "pid": 0x6111, "device": "COM2"}, + ], + ], + indirect=True, +) def test_invalid_led_value_raises_error(mock_serial_fixture): """ Test that an invalid LED value raises an error. """ mock_serial, mock_comports, mock_serial_instance = mock_serial_fixture - target = "LED" + key = "LED" value = 150 # LED value should be between 0 and 100 device_id = 1 port = "COM3" with pytest.raises(ValidationError, match="For LED, value must be between 0 and 100"): - device_update(target, value, device_id, port) - -@pytest.mark.parametrize('mock_serial_fixture', [ - [{'vid': 0x0403, 'pid': 0x6001, 'device': 'COM3'}, - {'vid': 0x0403, 'pid': 0x6001, 'device': 'COM2'}], -], indirect=True) + device_update(key, value, device_id, port) + + +@pytest.mark.parametrize( + "mock_serial_fixture", + [ + [ + {"vid": 0x0403, "pid": 0x6001, "device": "COM3"}, + {"vid": 0x0403, "pid": 0x6001, "device": "COM2"}, + ], + ], + indirect=True, +) def test_devupdate_with_multiple_ftdi_devices(mock_serial_fixture): """ Test that multiple FTDI devices raise an error. """ - target = "GAIN" + key = "GAIN" value = 5 device_id = 1 with pytest.raises(ValueError, match="Multiple FTDI devices found. Please specify the port."): - device_update(target, value, device_id) + device_update(key, value, device_id) -@pytest.mark.parametrize('mock_serial_fixture', [ - [{'vid': 0x0403, 'pid': 0x6001, 'device': 'COM3'}], -], indirect=True) + +@pytest.mark.parametrize( + "mock_serial_fixture", + [ + [{"vid": 0x0403, "pid": 0x6001, "device": "COM3"}], + ], + indirect=True, +) def test_devupdate_serial_exception_handling(mock_serial_fixture): """ Test exception handling when serial port cannot be opened. @@ -144,26 +194,31 @@ def test_devupdate_serial_exception_handling(mock_serial_fixture): mock_serial.side_effect = serial.SerialException("Serial port error") - target = "LED" + key = "LED" value = 50 device_id = 1 port = "COM3" with pytest.raises(serial.SerialException): - device_update(target, value, device_id, port) + device_update(key, value, device_id, port) + -@pytest.mark.parametrize('mock_serial_fixture', [ - [{'vid': 0x0413, 'pid': 0x6111, 'device': 'COM2'}], -], indirect=True) +@pytest.mark.parametrize( + "mock_serial_fixture", + [ + [{"vid": 0x0413, "pid": 0x6111, "device": "COM2"}], + ], + indirect=True, +) def test_specified_port_not_ftdi_device(mock_serial_fixture): """ Test with a specified port not corresponding to an FTDI device. """ mock_serial, mock_comports, mock_serial_instance = mock_serial_fixture - target = "GAIN" + key = "GAIN" value = 10 device_id = 1 with pytest.raises(ValueError, match="No FTDI devices found."): - device_update(target, value, device_id) + device_update(key, value, device_id) diff --git a/tests/test_formats.py b/tests/test_formats.py deleted file mode 100644 index 53b481d3..00000000 --- a/tests/test_formats.py +++ /dev/null @@ -1,49 +0,0 @@ -import pdb - -import pytest -import json -import importlib - -from miniscope_io.formats import WireFreeSDLayout - - -# More formats can be added here as needed. -@pytest.mark.parametrize('format', [WireFreeSDLayout]) -def test_to_from_json(format): - """ - A format can be exported and re-imported from JSON and remain equivalent - """ - fmt_json = format.model_dump_json() - - # convert the json to a dict - fmt_dict = json.loads(fmt_json) - - # Get the parent class - parent_class = type(format) - #parent_class_name = parent_module_str.split('.')[-1] - #parent_class = getattr(importlib.import_module(parent_module_str), parent_class_name) - - new_format = parent_class(**fmt_dict) - - assert format == new_format - - -@pytest.mark.parametrize( - ['format', 'format_json'], - [ - (WireFreeSDLayout, '{"sectors": {"header": 1022, "config": 1023, "data": 1024, "size": 512}, "write_key0": 226277911, "write_key1": 226277911, "write_key2": 226277911, "write_key3": 226277911, "word_size": 4, "header": {"gain": 4, "led": 5, "ewl": 6, "record_length": 7, "fs": 8, "delay_start": 9, "battery_cutoff": 10}, "config": {"width": 0, "height": 1, "fs": 2, "buffer_size": 3, "n_buffers_recorded": 4, "n_buffers_dropped": 5}, "buffer": {"length": 0, "linked_list": 1, "frame_num": 2, "buffer_count": 3, "frame_buffer_count": 4, "write_buffer_count": 5, "dropped_buffer_count": 6, "timestamp": 7, "data_length": 8, "write_timestamp": null, "battery_voltage": null}, "version": "0.1.1"}') - ] -) -def test_format_unchanged(format, format_json): - """ - A format is a constant and shouldn't change! - - This protects against changes in the parent classes breaking the formats, - and also breaking the formats themselves - """ - parent_class = type(format) - - format_dict = json.loads(format_json) - new_format = parent_class(**format_dict) - - assert new_format == format diff --git a/tests/test_io.py b/tests/test_io.py index a2c579c8..27c67a1b 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -7,17 +7,16 @@ import numpy as np import warnings -from miniscope_io.models.data import Frame -from miniscope_io.models.sdcard import SDBufferHeader -from miniscope_io.formats import WireFreeSDLayout, WireFreeSDLayout_Battery -from miniscope_io.io import SDCard -from miniscope_io.io import BufferedCSVWriter -from miniscope_io.exceptions import EndOfRecordingException -from miniscope_io.models.data import Frame -from miniscope_io.utils import hash_file, hash_video +from mio.models.sdcard import SDBufferHeader +from mio.io import SDCard +from mio.io import BufferedCSVWriter +from mio.exceptions import EndOfRecordingException +from mio.models.data import Frame +from mio.utils import hash_file, hash_video from .fixtures import wirefree, wirefree_battery + @pytest.fixture def tmp_csvfile(tmp_path): """ @@ -25,6 +24,7 @@ def tmp_csvfile(tmp_path): """ return tmp_path / "test.csv" + def test_csvwriter_initialization(tmp_csvfile): """ Test that the BufferedCSVWriter initializes correctly. @@ -34,6 +34,7 @@ def test_csvwriter_initialization(tmp_csvfile): assert writer.buffer_size == 10 assert writer.buffer == [] + def test_csvwriter_append_and_flush(tmp_csvfile): """ Test that the BufferedCSVWriter appends to the buffer and flushes it when full. @@ -41,16 +42,17 @@ def test_csvwriter_append_and_flush(tmp_csvfile): writer = BufferedCSVWriter(tmp_csvfile, buffer_size=2) writer.append([1, 2, 3]) assert len(writer.buffer) == 1 - + writer.append([4, 5, 6]) assert len(writer.buffer) == 0 assert tmp_csvfile.exists() - - with tmp_csvfile.open('r', newline='') as f: + + with tmp_csvfile.open("r", newline="") as f: reader = csv.reader(f) rows = list(reader) assert len(rows) == 2 - assert rows == [['1', '2', '3'], ['4', '5', '6']] + assert rows == [["1", "2", "3"], ["4", "5", "6"]] + def test_csvwriter_flush_buffer(tmp_csvfile): """ @@ -59,15 +61,16 @@ def test_csvwriter_flush_buffer(tmp_csvfile): writer = BufferedCSVWriter(tmp_csvfile, buffer_size=2) writer.append([1, 2, 3]) writer.flush_buffer() - + assert len(writer.buffer) == 0 assert tmp_csvfile.exists() - - with tmp_csvfile.open('r', newline='') as f: + + with tmp_csvfile.open("r", newline="") as f: reader = csv.reader(f) rows = list(reader) assert len(rows) == 1 - assert rows == [['1', '2', '3']] + assert rows == [["1", "2", "3"]] + def test_csvwriter_close(tmp_csvfile): """ @@ -76,15 +79,16 @@ def test_csvwriter_close(tmp_csvfile): writer = BufferedCSVWriter(tmp_csvfile, buffer_size=2) writer.append([1, 2, 3]) writer.close() - + assert len(writer.buffer) == 0 assert tmp_csvfile.exists() - - with tmp_csvfile.open('r', newline='') as f: + + with tmp_csvfile.open("r", newline="") as f: reader = csv.reader(f) rows = list(reader) assert len(rows) == 1 - assert rows == [['1', '2', '3']] + assert rows == [["1", "2", "3"]] + def test_read(wirefree): """ @@ -169,7 +173,7 @@ def test_relative_path(): rel_path = abs_child.relative_to(abs_cwd) assert not rel_path.is_absolute() - sdcard = SDCard(drive=rel_path, layout=WireFreeSDLayout) + sdcard = SDCard(drive=rel_path, layout="wirefree-sd-layout") # check we can do something basic like read config assert sdcard.config is not None @@ -180,7 +184,7 @@ def test_relative_path(): # now try with an absolute path abs_path = rel_path.resolve() assert abs_path.is_absolute() - sdcard_abs = SDCard(drive=abs_path, layout=WireFreeSDLayout) + sdcard_abs = SDCard(drive=abs_path, layout="wirefree-sd-layout") assert sdcard_abs.config is not None assert sdcard_abs.drive.is_absolute() @@ -212,7 +216,7 @@ def test_to_img(wirefree_battery, n_frames, hash, tmp_path): assert out_hash == hash - sd = SDCard(out_file, WireFreeSDLayout_Battery) + sd = SDCard(out_file, "wirefree-sd-layout-battery") # we should be able to read all the frames! frames = [] diff --git a/tests/test_logging.py b/tests/test_logging.py index b528163f..13fcc3bd 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -1,11 +1,26 @@ -import pdb +import logging import pytest -import os -import tempfile from pathlib import Path +import re +import multiprocessing as mp +from time import sleep +import warnings + +from logging.handlers import RotatingFileHandler +from rich.logging import RichHandler + +from mio.logging import init_logger + + +@pytest.fixture(autouse=True) +def reset_root_logger(): + """ + Before each test, reset the root logger + """ + root_logger = logging.getLogger("mio") + root_logger.handlers.clear() -from miniscope_io.logging import init_logger def test_init_logger(capsys, tmp_path): """ @@ -14,33 +29,147 @@ def test_init_logger(capsys, tmp_path): - with separable levels """ - log_dir = Path(tmp_path) / 'logs' + log_dir = Path(tmp_path) / "logs" log_dir.mkdir() - log_file = log_dir / 'miniscope_io.test_logger.log' - logger = init_logger( - name='test_logger', - log_dir=log_dir, - level='INFO', - file_level='WARNING' - ) - warn_msg = 'Both loggers should show' + log_file = log_dir / "mio.log" + logger = init_logger(name="test_logger", log_dir=log_dir, level="INFO", file_level="WARNING") + warn_msg = "Both loggers should show" logger.warning(warn_msg) # can't test for presence of string because logger can split lines depending on size of console # but there should be one WARNING in stdout captured = capsys.readouterr() - assert 'WARNING' in captured.out + assert "WARNING" in captured.out - with open(log_file, 'r') as lfile: + with open(log_file, "r") as lfile: log_str = lfile.read() - assert 'WARNING' in log_str + assert "WARNING" in log_str info_msg = "Now only stdout should show" logger.info(info_msg) captured = capsys.readouterr() - assert 'INFO' in captured.out - with open(log_file, 'r') as lfile: + assert "INFO" in captured.out + with open(log_file, "r") as lfile: log_str = lfile.read() - assert 'INFO' not in log_str + assert "INFO" not in log_str + + +def test_nested_loggers(capsys, tmp_path): + """ + Nested loggers should not double-log + """ + log_dir = Path(tmp_path) + + parent = init_logger("parent", log_dir=log_dir, level="DEBUG", file_level="DEBUG") + child = init_logger("parent.child", log_dir=log_dir, level="DEBUG", file_level="DEBUG") + + child.debug("hey") + parent.debug("sup") + + root_logger = logging.getLogger("mio") + + warnings.warn(f"FILES IN LOG DIR: {list(log_dir.glob('*'))}") + warnings.warn(f"ROOT LOGGER HANDLERS: {root_logger.handlers}") + + assert len(root_logger.handlers) == 2 + assert len(parent.handlers) == 0 + assert len(child.handlers) == 0 + + with open(log_dir / "mio.log") as lfile: + file_logs = lfile.read() + + # only one message of each! + stdout = capsys.readouterr() + assert len(re.findall("hey", stdout.out)) == 1 + assert len(re.findall("sup", stdout.out)) == 1 + assert len(re.findall("hey", file_logs)) == 1 + assert len(re.findall("sup", file_logs)) == 1 + + +@pytest.mark.parametrize("level", ["DEBUG", "INFO", "WARNING", "ERROR"]) +@pytest.mark.parametrize("direct_setting", [True, False]) +@pytest.mark.parametrize("test_target", ["logger", "RotatingFileHandler", "RichHandler"]) +def test_init_logger_from_config( + tmp_path, monkeypatch, level, direct_setting, test_target, set_config +): + """ + Set log levels from all kinds of config + """ + # Feels kind of fragile to hardcode this but I couldn't think of a better way so for now + level_name_map = { + "DEBUG": logging.DEBUG, + "INFO": logging.INFO, + "WARNING": logging.WARNING, + "ERROR": logging.ERROR, + } + + if direct_setting: + set_config({"logs": {"level_file": level, "level_stdout": level}}) + else: + set_config({"logs": {"level": level}}) + + dotenv_logger = init_logger(name="test_logger", log_dir=tmp_path) + root_logger = logging.getLogger("mio") + + # Separating them for readable summary info + if test_target == "logger": + assert dotenv_logger.level == level_name_map.get(level) + + assert len(dotenv_logger.handlers) == 0 + assert len(root_logger.handlers) == 2 + file_handlers = [h for h in root_logger.handlers if isinstance(h, RotatingFileHandler)] + stream_handlers = [h for h in root_logger.handlers if isinstance(h, RichHandler)] + assert len(file_handlers) == 1 + assert len(stream_handlers) == 1 + file_handler = file_handlers[0] + stream_handler = stream_handlers[0] + + if test_target == "RotatingFileHandler": + assert file_handler.level == level_name_map.get(level) + + elif test_target == "RichHandler": + # Might be better to explicitly set the level in the handler + assert stream_handler.level == level_name_map.get(level) + + +def _mp_function(name, path): + logger = init_logger(name, log_dir=path, level="DEBUG", file_level="DEBUG") + for i in range(100): + sleep(0.001) + logger.debug(f"{name} - {i}") + + +def test_multiprocess_logging(capfd, tmp_path): + """ + We should be able to handle logging from multiple processes + """ + + proc_1 = mp.Process(target=_mp_function, args=("proc_1", tmp_path)) + proc_2 = mp.Process(target=_mp_function, args=("proc_2", tmp_path)) + proc_3 = mp.Process(target=_mp_function, args=("proc_1.proc_3", tmp_path)) + + proc_1.start() + proc_2.start() + proc_3.start() + proc_1.join() + proc_2.join() + proc_3.join() + + stdout = capfd.readouterr() + logs = {} + for log_file in tmp_path.glob("*.log"): + with open(log_file) as lfile: + logs[log_file.name] = lfile.read() + + assert "mio.log" in logs + assert len(logs) == 4 + + for logfile, logs in logs.items(): + # main logfile does not receive messages + if logfile == "mio.log": + assert len(logs.split("\n")) == 1 + else: + assert len(logs.split("\n")) == 101 + assert len(re.findall("DEBUG", stdout.out)) == 300 diff --git a/tests/test_mixins.py b/tests/test_mixins.py new file mode 100644 index 00000000..7976ecbe --- /dev/null +++ b/tests/test_mixins.py @@ -0,0 +1,157 @@ +from pathlib import Path +from importlib.metadata import version + +import pytest +import yaml +from pydantic import BaseModel, ConfigDict + +from mio import CONFIG_DIR +from mio.models.mixins import yaml_peek, ConfigYAMLMixin +from tests.fixtures import tmp_config_source, yaml_config + + +class NestedModel(BaseModel): + d: int = 4 + e: str = "5" + f: float = 5.5 + + +class MyModel(ConfigYAMLMixin): + id: str = "my-config" + a: int = 0 + b: str = "1" + c: float = 2.2 + child: NestedModel = NestedModel() + + +class LoaderModel(ConfigYAMLMixin): + """Model that just allows everything, only used to test write on load""" + + model_config = ConfigDict(extra="allow") + + +@pytest.mark.parametrize( + "id,path,valid", + [ + ("default-path", None, True), + ("nested-path", Path("configs/nested/path/config.yaml"), True), + ("not-valid", Path("not_in_dir/config.yaml"), False), + ], +) +def test_config_from_id(yaml_config, id, path, valid): + """Configs can be looked up with the id field if they're within a config directory""" + instance = MyModel(id=id) + yaml_config(id, instance.model_dump(), path) + if valid: + loaded = MyModel.from_id(id) + assert loaded == instance + assert loaded.child == instance.child + assert isinstance(loaded.child, NestedModel) + else: + with pytest.raises(KeyError): + MyModel.from_id(id) + + +def test_roundtrip_to_from_yaml(tmp_config_source): + """Config models can roundtrip to and from yaml""" + yaml_file = tmp_config_source / "test_config.yaml" + + instance = MyModel() + instance.to_yaml(yaml_file) + loaded = MyModel.from_yaml(yaml_file) + assert loaded == instance + assert loaded.child == instance.child + assert isinstance(loaded.child, NestedModel) + + +@pytest.mark.parametrize( + "src", + [ + pytest.param( + """ +a: 9 +b: "10\"""", + id="missing", + ), + pytest.param( + f""" +a: 9 +id: "my-config" +mio_model: "tests.test_mixins.MyModel" +mio_version: "{version('mio')}" +b: "10\"""", + id="not-at-start", + ), + pytest.param( + f""" +mio_version: "{version('mio')}" +mio_model: "tests.test_mixins.MyModel" +id: "my-config" +a: 9 +b: "10\"""", + id="out-of-order", + ), + ], +) +def test_complete_header(tmp_config_source, src: str): + """ + Config models saved without header information will have it filled in + the source yaml they were loaded from + """ + yaml_file = tmp_config_source / "test_config.yaml" + + with open(yaml_file, "w") as yfile: + yfile.write(src) + + _ = MyModel.from_yaml(yaml_file) + + with open(yaml_file, "r") as yfile: + loaded = yaml.safe_load(yfile) + + loaded_str = yaml_file.read_text() + + assert loaded["mio_version"] == version("mio") + assert loaded["id"] == "my-config" + assert loaded["mio_model"] == MyModel._model_name() + + # the header should come at the top! + lines = loaded_str.splitlines() + for i, key in enumerate(("id", "mio_model", "mio_version")): + line_key = lines[i].split(":")[0].strip() + assert line_key == key + + +@pytest.mark.parametrize("config_file", CONFIG_DIR.rglob("*.y*ml")) +def test_builtins_unchanged(config_file): + """None of the builtin configs should be modified on load - i.e. they should all have correct headers.""" + before = config_file.read_text() + _ = LoaderModel.from_yaml(config_file) + after = config_file.read_text() + assert ( + before == after + ), f"Packaged config {config_file} was modified on load, ensure it has the correct headers." + + +@pytest.mark.parametrize( + "key,expected,root,first", + [ + ("key1", "val1", True, True), + ("key1", "val1", False, True), + ("key1", ["val1"], True, False), + ("key1", ["val1", "val2"], False, False), + ("key2", "val2", True, True), + ("key3", False, True, True), + ("key4", False, True, True), + ("key4", "val4", False, True), + ], +) +def test_peek_yaml(key, expected, root, first, yaml_config): + yaml_file = yaml_config( + "test", {"key1": "val1", "key2": "val2", "key3": {"key1": "val2", "key4": "val4"}}, None + ) + + if not expected: + with pytest.raises(KeyError): + _ = yaml_peek(key, yaml_file, root=root, first=first) + else: + assert yaml_peek(key, yaml_file, root=root, first=first) == expected diff --git a/tests/test_models/test_model_buffer.py b/tests/test_models/test_model_buffer.py index 6811a424..be46e3c5 100644 --- a/tests/test_models/test_model_buffer.py +++ b/tests/test_models/test_model_buffer.py @@ -1,5 +1,5 @@ import pytest -from miniscope_io.models.buffer import BufferHeader, BufferHeaderFormat +from mio.models.buffer import BufferHeader, BufferHeaderFormat from pydantic import ValidationError @@ -10,6 +10,7 @@ def test_buffer_from_format(construct): Instantiate a BufferHeader from a sequence and a format """ format = BufferHeaderFormat( + id="buffer-header", linked_list=0, frame_num=1, buffer_count=2, @@ -25,7 +26,7 @@ def test_buffer_from_format(construct): # correct vals should work in both cases instance = BufferHeader.from_format(vals, format, construct) - assert list(instance.model_dump().values()) == vals + assert list(instance.model_dump(exclude={"id"}).values()) == vals # bad vals should only work if we're constructing if construct: diff --git a/tests/test_models/test_model_devupdate.py b/tests/test_models/test_model_devupdate.py index f5a8e8ef..9abc0f85 100644 --- a/tests/test_models/test_model_devupdate.py +++ b/tests/test_models/test_model_devupdate.py @@ -2,7 +2,8 @@ from unittest.mock import patch from pydantic import ValidationError -from miniscope_io.models.devupdate import DevUpdateCommand, UpdateTarget, DeviceCommand +from mio.models.devupdate import DevUpdateCommand, UpdateKey, DeviceCommand + def mock_comports(): class Port: @@ -11,38 +12,46 @@ def __init__(self, device): return [Port("COM1"), Port("COM2")] + @pytest.fixture def mock_serial_ports(): - with patch('serial.tools.list_ports.comports', side_effect=mock_comports): + with patch("serial.tools.list_ports.comports", side_effect=mock_comports): yield + def test_valid_led_update(mock_serial_ports): - cmd = DevUpdateCommand(device_id=1, port="COM1", target="LED", value=50) - assert cmd.target == UpdateTarget.LED + cmd = DevUpdateCommand(device_id=1, port="COM1", key="LED", value=50) + assert cmd.key == UpdateKey.LED assert cmd.value == 50 + def test_valid_gain_update(mock_serial_ports): - cmd = DevUpdateCommand(device_id=1, port="COM2", target="GAIN", value=2) - assert cmd.target == UpdateTarget.GAIN + cmd = DevUpdateCommand(device_id=1, port="COM2", key="GAIN", value=2) + assert cmd.key == UpdateKey.GAIN assert cmd.value == 2 + def test_invalid_led_value(mock_serial_ports): with pytest.raises(ValidationError): - DevUpdateCommand(device_id=1, port="COM1", target="LED", value=150) + DevUpdateCommand(device_id=1, port="COM1", key="LED", value=150) + def test_invalid_gain_value(mock_serial_ports): with pytest.raises(ValidationError): - DevUpdateCommand(device_id=1, port="COM1", target="GAIN", value=3) + DevUpdateCommand(device_id=1, port="COM1", key="GAIN", value=3) + -def test_invalid_target(mock_serial_ports): +def test_invalid_key(mock_serial_ports): with pytest.raises(ValueError): - DevUpdateCommand(device_id=1, port="COM1", target="FAKEDEVICE", value=10) + DevUpdateCommand(device_id=1, port="COM1", key="FAKEDEVICE", value=10) + def test_invalid_port(): - with patch('serial.tools.list_ports.comports', return_value=mock_comports()): + with patch("serial.tools.list_ports.comports", return_value=mock_comports()): with pytest.raises(ValidationError): - DevUpdateCommand(device_id=1, port="COM3", target="LED", value=50) + DevUpdateCommand(device_id=1, port="COM3", key="LED", value=50) + def test_device_command(mock_serial_ports): - cmd = DevUpdateCommand(device_id=1, port="COM2", target="DEVICE", value=DeviceCommand.REBOOT.value) - assert cmd.value == DeviceCommand.REBOOT.value \ No newline at end of file + cmd = DevUpdateCommand(device_id=1, port="COM2", key="DEVICE", value=DeviceCommand.REBOOT.value) + assert cmd.value == DeviceCommand.REBOOT.value diff --git a/tests/test_models/test_model_mixins.py b/tests/test_models/test_model_mixins.py index 105ad150..2abe3b86 100644 --- a/tests/test_models/test_model_mixins.py +++ b/tests/test_models/test_model_mixins.py @@ -3,27 +3,24 @@ import yaml from pydantic import BaseModel -from miniscope_io.models.mixins import YAMLMixin +from mio.models.mixins import YAMLMixin + def test_yaml_mixin(tmp_path): """ YAMLMixIn should give our models a from_yaml method to read from files """ + class MyModel(BaseModel, YAMLMixin): a_str: str a_int: int a_list: List[int] a_dict: Dict[str, float] - data = { - 'a_str': 'string!', - 'a_int': 5, - 'a_list': [1,2,3], - 'a_dict': {'a': 1.1, 'b': 2.5} - } + data = {"a_str": "string!", "a_int": 5, "a_list": [1, 2, 3], "a_dict": {"a": 1.1, "b": 2.5}} - yaml_file = tmp_path / 'temp.yaml' - with open(yaml_file, 'w') as yfile: + yaml_file = tmp_path / "temp.yaml" + with open(yaml_file, "w") as yfile: yaml.safe_dump(data, yfile) instance = MyModel.from_yaml(yaml_file) diff --git a/tests/test_models/test_model_stream.py b/tests/test_models/test_model_stream.py index f938181a..e53f5deb 100644 --- a/tests/test_models/test_model_stream.py +++ b/tests/test_models/test_model_stream.py @@ -1,7 +1,7 @@ import pytest -from miniscope_io import DATA_DIR -from miniscope_io.models.stream import ADCScaling, StreamDevConfig, StreamBufferHeader +from mio import DATA_DIR +from mio.models.stream import ADCScaling, StreamDevConfig, StreamBufferHeader from ..conftest import CONFIG_DIR diff --git a/tests/test_sdcard.py b/tests/test_sdcard.py index a64da000..7a3a8338 100644 --- a/tests/test_sdcard.py +++ b/tests/test_sdcard.py @@ -1,15 +1,16 @@ import pytest -from miniscope_io.models.sdcard import SectorConfig +from mio.models.sdcard import SectorConfig import numpy as np + @pytest.fixture def random_sectorconfig(): return SectorConfig( header=np.random.randint(0, 2048), config=np.random.randint(0, 2048), - data=np.random.randint(0,2048), - size=np.random.randint(0,2048) + data=np.random.randint(0, 2048), + size=np.random.randint(0, 2048), ) diff --git a/tests/test_stream_daq.py b/tests/test_stream_daq.py index 41febdfb..d10a29b7 100644 --- a/tests/test_stream_daq.py +++ b/tests/test_stream_daq.py @@ -1,20 +1,25 @@ import re from pathlib import Path +import multiprocessing +import os import pytest import pandas as pd - -from miniscope_io import BASE_DIR -from miniscope_io.stream_daq import StreamDevConfig, StreamDaq -from miniscope_io.utils import hash_video, hash_file +import sys +import signal +import time +from contextlib import contextmanager + +from mio import BASE_DIR +from mio.stream_daq import StreamDevConfig, StreamDaq +from mio.utils import hash_video, hash_file from .conftest import DATA_DIR, CONFIG_DIR @pytest.fixture(params=[pytest.param(5, id="buffer-size-5"), pytest.param(10, id="buffer-size-10")]) def default_streamdaq(set_okdev_input, request) -> StreamDaq: - test_config_path = CONFIG_DIR / "stream_daq_test_200px.yml" - daqConfig = StreamDevConfig.from_yaml(test_config_path) + daqConfig = StreamDevConfig.from_id("test-wireless-200px") daqConfig.runtime.frame_buffer_queue_size = request.param daqConfig.runtime.image_buffer_queue_size = request.param daqConfig.runtime.serial_buffer_queue_size = request.param @@ -31,12 +36,14 @@ def default_streamdaq(set_okdev_input, request) -> StreamDaq: "config,data,video_hash_list,show_video", [ ( - "stream_daq_test_200px.yml", + "test-wireless-200px", "stream_daq_test_fpga_raw_input_200px.bin", [ "f878f9c55de28a9ae6128631c09953214044f5b86504d6e5b0906084c64c644c", "8a6f6dc69275ec3fbcd69d1e1f467df8503306fa0778e4b9c1d41668a7af4856", "3676bc4c6900bc9ec18b8387abdbed35978ebc48408de7b1692959037bc6274d", + "3891091fd2c1c59b970e7a89951aeade8ae4eea5627bee860569a481bfea39b7", + "d8e519c1d7e74cdebc39f11bb5c7e189011f025410a0746af7aa34bdb2e72e8e", ], False, ) @@ -47,8 +54,7 @@ def test_video_output( ): output_video = tmp_path / "output.avi" - test_config_path = CONFIG_DIR / config - daqConfig = StreamDevConfig.from_yaml(test_config_path) + daqConfig = StreamDevConfig.from_id(config) daqConfig.runtime.frame_buffer_queue_size = buffer_size daqConfig.runtime.image_buffer_queue_size = buffer_size daqConfig.runtime.serial_buffer_queue_size = buffer_size @@ -70,14 +76,13 @@ def test_video_output( "config,data", [ ( - "stream_daq_test_200px.yml", + "test-wireless-200px", "stream_daq_test_fpga_raw_input_200px.bin", ) ], ) def test_binary_output(config, data, set_okdev_input, tmp_path): - test_config_path = CONFIG_DIR / config - daqConfig = StreamDevConfig.from_yaml(test_config_path) + daqConfig = StreamDevConfig.from_id(config) data_file = DATA_DIR / data set_okdev_input(data_file) @@ -106,7 +111,7 @@ def test_csv_output(tmp_path, default_streamdaq, write_metadata, caplog): # actually not sure what we should be looking for here, for now we just check for shape # this should be the same as long as the test data stays the same, # but it's a pretty weak test. - assert df.shape == (910, 11) + assert df.shape == (910, 12) # ensure there were no errors during capture for record in caplog.records: @@ -116,6 +121,48 @@ def test_csv_output(tmp_path, default_streamdaq, write_metadata, caplog): assert not output_csv.exists() +# This is a helper function for test_continuous_and_termination() that is currently skipped +""" +def capture_wrapper(default_streamdaq, source, show_video, continuous): + try: + default_streamdaq.capture(source=source, show_video=show_video, continuous=continuous) + except KeyboardInterrupt: + pass # expected +""" + + +@pytest.mark.skip( + "Needs to be implemented. Temporary skipped because tests fail in some OS (See GH actions)." +) +@pytest.mark.timeout(10) +def test_continuous_and_termination(tmp_path, default_streamdaq): + """ + Make sure continuous mode runs forever until interrupted, and that all processes are + cleaned up when the capture process is terminated. + """ + """ + timeout = 1 + + capture_process = multiprocessing.Process(target=capture_wrapper, args=(default_streamdaq, "fpga", False, True)) + + capture_process.start() + alive_processes = default_streamdaq.alive_processes() + initial_alive_processes = len(alive_processes) + + time.sleep(timeout) + + alive_processes = default_streamdaq.alive_processes() + assert len(alive_processes) == initial_alive_processes + + os.kill(capture_process.pid, signal.SIGINT) + capture_process.join() + + alive_processes = default_streamdaq.alive_processes() + assert len(alive_processes) == 0 + """ + pass + + def test_metadata_plotting(tmp_path, default_streamdaq): """ Setting the capture kwarg ``show_metadata == True`` should plot the frame metadata