Combined Pipeline

.github/workflows/tests.yml

@@ -8,6 +8,7 @@ on:
   pull_request:
     branches:
       - main
+      - pipeline
 
 # Don't run duplicates - eg. if pushing to an open PR
 #concurrency:
@@ -22,7 +23,7 @@ jobs:
       fail-fast: false
       matrix:
         os: ['ubuntu-latest', 'windows-latest', 'macos-latest']
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12"]
 
     steps:
       - uses: actions/checkout@v3

docs/api/devices/camera.md

@@ -0,0 +1,7 @@
+# camera
+
+```{eval-rst}
+.. automodule:: mio.devices.camera
+    :members:
+    :undoc-members:
+```

docs/api/devices/index.md

@@ -0,0 +1,13 @@
+# devices
+
+```{toctree}
+:max-depth: 2
+
+camera
+```
+
+```{eval-rst}
+.. automodule:: mio.devices.device
+    :members:
+    :undoc-members:
+```

docs/api/devices/wirefree.md

@@ -0,0 +1,7 @@
+# wirefree
+
+```{eval-rst}
+.. automodule:: mio.devices.wirefree
+    :members:
+    :undoc-members:
+```

docs/api/models/index.md

@@ -16,11 +16,14 @@ keep what is common common, and what is unique unique.
 ```
 
 ```{toctree}
+:maxdepth: 1
+
 buffer
 config
 data
 mixins
 models
+pipeline
 sdcard
 stream
 ```

docs/api/models/pipeline.md

@@ -0,0 +1,7 @@
+# pipeline
+
+```{eval-rst}
+.. automodule:: mio.models.pipeline
+    :members:
+    :undoc-members:
+```

docs/api/sources.md

@@ -0,0 +1,11 @@
+# sources
+
+Interfaces to external source devices like miniscopes and DAQs
+
+## OpalKelly
+
+```{eval-rst}
+.. autoclass:: mio.sources.opalkelly.okDev
+    :members:
+    :undoc-members:
+```

docs/conf.py

@@ -6,6 +6,7 @@
 # -- Project information -----------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
 
+from pathlib import Path
 from importlib.metadata import version as _version
 import sys
 from unittest.mock import Mock
@@ -27,12 +28,14 @@
     "myst_parser",
     "sphinx.ext.napoleon",
     "sphinx.ext.autodoc",
-    "sphinxcontrib.autodoc_pydantic",
-    "sphinxcontrib.programoutput",
+    "sphinx.ext.inheritance_diagram",
     "sphinx.ext.intersphinx",
     "sphinx.ext.todo",
     "sphinx_click",
     "sphinx_design",
+    "sphinxcontrib.autodoc_pydantic",
+    "sphinxcontrib.mermaid",
+    "sphinxcontrib.programoutput",
 ]
 
 templates_path = ["_templates"]
@@ -84,3 +87,56 @@
 
 # todo
 todo_include_todos = True
+
+# mermaid
+
+mermaid_include_elk = "latest"
+mermaid_version = None
+mermaid_use_local = "https://example.com"
+mermaid_init_js = """
+import mermaid from 'https://cdn.jsdelivr.net/npm/mermaid/dist/mermaid.esm.min.mjs'
+import elkLayouts from 'https://cdn.jsdelivr.net/npm/@mermaid-js/layout-elk/dist/mermaid-layout-elk.esm.min.mjs';
+mermaid.registerLayoutLoaders(elkLayouts);
+const make_config = () => {
+  let prefersDark = localStorage.getItem('theme') === 'dark' || (localStorage.getItem('theme') === null && window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches)
+  return({
+    startOnLoad:false,
+    darkMode: prefersDark,
+    theme: prefersDark ? "dark" : "default"
+  })
+}
+
+const init_mermaid = () => {
+    let graphs = document.querySelectorAll(".mermaid");
+    [...graphs].forEach((element) => {
+        if (!element.hasAttribute("data-source")) {
+            element.setAttribute("data-source", element.innerText);
+        }
+        if (element.hasAttribute("data-processed")) {
+            let new_elt = document.createElement("pre");
+            let graph_source = element.getAttribute("data-source");
+            new_elt.appendChild(document.createTextNode(graph_source));
+            new_elt.classList.add("mermaid");
+            new_elt.setAttribute("data-source", graph_source);
+            element.replaceWith(new_elt);
+        }
+    });
+
+    let config = make_config()
+    mermaid.initialize(config);
+    mermaid.run();
+}
+
+init_mermaid();
+
+let theme_observer = new MutationObserver(init_mermaid);
+let body = document.getElementsByTagName("body")[0];
+theme_observer.observe(body, {attributes: true});
+window.theme_observer = theme_observer;
+
+"""
+
+# inheritance graphs
+inheritance_graph_attrs = {
+    "rankdir": "TB",
+}

docs/guide/capture/index.md

@@ -0,0 +1,5 @@
+# Capture Data
+
+```{toctree}
+wireless
+```

docs/device/test_device.md → docs/guide/capture/wireless.md

@@ -1,18 +1,18 @@
-# Example stream device
+# Capture from a Wireless Miniscope
 
 **Under Construction:** This section will be populated when devices are released.
 
 ## Buffer Structure
 - **Contents**: The buffer consists of a concatenation of 32-bit dummy words, 32-bit header data, and 8-bit pixel data. The dummy words are for stabilizing the clock recovery function in the Manchester decoder FPGA, and the header data is used for recovering images and detecting device status.
-- **Payload**: A single image is split and stored in a circulating buffer within the device. The [`num_buffers`](../api/stream_daq.md) should match the number of circulating buffers in the device.
+- **Payload**: A single image is split and stored in a circulating buffer within the device. The [`num_buffers`](../../api/stream_daq.md) should match the number of circulating buffers in the device.
 
 ## Header Values and Expected Transitions
 See following docs for the basic structure.
 - `mio.models.buffer.BufferHeaderFormat`
 - `mio.models.stream.StreamBufferHeaderFormat`
 
 Device specific notes are listed below.
-- **`preamble`**: 32-bit preamble for detecting the beginning of each buffer. The [`preamble`](../api/stream_daq.md) in the device config needs to match the preamble defined in firmware.
+- **`preamble`**: 32-bit preamble for detecting the beginning of each buffer. The [`preamble`](../../api/stream_daq.md) in the device config needs to match the preamble defined in firmware.
 - **`dropped_buffer_count`**: Currently not used and should always be zero.
 - **`write_timestamp`**: Currently not used and should always be zero.
 - **`battery_voltage`**: Currently not used and should always be zero.

docs/guide/update/index.md

@@ -0,0 +1,5 @@
+# Update Devices
+
+```{toctree}
+wireless
+```

docs/device/update_controller.md → docs/guide/update/wireless.md

@@ -1,4 +1,4 @@
-# Update controller
+# Update A Wireless Miniscope
 
 **Under Construction:** This section will be populated when these devices are released.
 

docs/index.md

@@ -8,27 +8,29 @@ Generic I/O interfaces for miniscopes :)
 
 guide/installation
 guide/config
+guide/capture/index
+guide/update/index
 cli/index
 ```
 
 ```{toctree}
-:caption: Device:
+:caption: Reference:
 :maxdepth: 1
 
-device/test_device
-device/update_controller
+reference/object_model
 ```
 
 ```{toctree}
 :maxdepth: 1
 :caption: API:
 
-api/devices
+api/devices/index
 api/io
 api/logging
 api/models/index
 api/exceptions
 api/plots/index
+api/sources
 api/utils
 api/stream_daq
 api/bit_operation

docs/meta/changelog.md

@@ -188,7 +188,7 @@ New features:
 
 - Added support for the wireless FPGA and UART daqs - work in progress unifying the API, but
   initial version of code is present in `stream_daq.py`
-- Vendored opalkelly device drivers - see `devices` and `vendor`
+- Vendored opalkelly device drivers - see `devices` (now `sources` in later versions) and `vendor`
 
 ### 0.1.5 - 23-09-03
 

docs/reference/object_model.md

@@ -0,0 +1,49 @@
+# Object Model
+
+## Devices
+
+A "{class}`.Device`" is the top-level class that one is expected to interact with when recording 
+data.
+
+It encapsulates a {class}`.Pipeline` object that parameterizes one of several
+{class}`.Source` nodes (like the {class}`.okDev`, {class]`.SDCard`, etc.),
+zero-to-several {class}`.Transform` nodes that transform the source ouput,
+and then one or several {class}`.Sink` nodes for saving data, plotting, and so on.
+
+```{autoclasstree} mio.devices.Device mio.devices.Camera mio.devices.Miniscope 
+:namespace: mio
+```  
+
+## Pipeline
+
+A {class}`.Pipeline` is the basic unit of data processing in miniscope-io.
+
+A {class}`.Pipeline` contains a set of {class}`.Node`s, which have three root types:
+
+- {class}`.Source` nodes have outputs but no inputs
+- {class}`.Sink` nodes have inputs but no outputs
+- {class}`.Transform` nodes have both inputs and outputs
+
+Nodes can be connected to any number of `input` or `output` nodes,
+and it is the responsibility of the {class}`.Pipeline` class to orchestrate
+their run: launching them in a threadqueue, passing data between them, etc.
+
+{class}`.Node`s should not directly call each other, and instead implement
+their {meth}`.Node.process` method as a "pure"-ish[^pureish] function that takes
+data as input and returns data as output without knowledge of the nodes
+that are connected to it.
+
+```{autoclasstree} mio.models.PipelineModel mio.models.Node mio.models.Source mio.models.Sink mio.models.ProcessingNode mio.models.Pipeline
+:namespace: mio
+```  
+
+## Config
+
+```{todo}
+Complete configuration documentation:
+- Sources of config - user dir, prepackaged, etc.
+- Types of config - device, session, etc.
+```
+
+[^pureish]: As in nodes can be stateful and use their instance attributes,
+  but should not rely on side-effects, implement their own queues, etc.

mio/__init__.py

@@ -11,7 +11,7 @@
 BASE_DIR = Path(__file__).parent.resolve()
 DATA_DIR = BASE_DIR / "data"
 CONFIG_DIR = DATA_DIR / "config"
-DEVICE_DIR = BASE_DIR / "devices"
+SOURCES_DIR = BASE_DIR / "sources"
 
 __all__ = [
     "BASE_DIR",

mio/devices/__init__.py

@@ -1,3 +1,16 @@
 """
-Control interfaces for external hardware devices
+Devices!
 """
+
+from mio.devices.camera import Camera, Miniscope, RecordingCameraMixin
+from mio.devices.device import Device, DeviceConfig
+from mio.devices.wirefree import WireFreeMiniscope
+
+__all__ = [
+    "Camera",
+    "Device",
+    "DeviceConfig",
+    "Miniscope",
+    "RecordingCameraMixin",
+    "WireFreeMiniscope",
+]