Merge pull request #45 from Aharoni-Lab/feature_continuous_run

Continuous mode / UNIX time log / Update termination logic

docs/meta/changelog.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## 0.5
+
+### 0.5.0 - 24-11-11
+Enhancements and bugfixes to `StreamDaq`; adding `device_update` module; CI/CD updates.
+
+#### Features / bugfixes
+- **Over-the-air device config:** modules and commands for updating and rebooting; *e.g.,* `mio update --key LED --value 10`, `mio device --reboot`.
+- **Continuous run:** updated error handling to continuously capture even when the data stream is interrupted.
+- **UNIX timestamp:** added UNIX timestamp to metadata file export.
+- **More Opal Kelly bitfiles:** added FPGA configuration images and organized them based on Manchester encoding conventions, frequency, etc.
+#### CI/CD
+- Switched to `pdm` from `poetry`; now `pdm install --with all` for contributing.
+- Added workflow for readthedocs preview link in PRs.
+- Added snake_case enforcement (Lint).
+
+Related PRs: [#45](https://github.com/Aharoni-Lab/miniscope-io/pull/45), [#48](https://github.com/Aharoni-Lab/miniscope-io/pull/48), [#49](https://github.com/Aharoni-Lab/miniscope-io/pull/49), [#50](https://github.com/Aharoni-Lab/miniscope-io/pull/50), [#53](https://github.com/Aharoni-Lab/miniscope-io/pull/53), 
+Contributors: [@t-sasatani](https://github.com/t-sasatani), [@sneakers-the-rat](https://github.com/sneakers-the-rat), [@MarcelMB](https://github.com/MarcelMB), [@phildong](https://github.com/phildong)
+
 ## 0.4
 
 ### 0.4.1 - 24-09-01

miniscope_io/cli/stream.py

@@ -56,6 +56,7 @@ def _capture_options(fn: Callable) -> Callable:
         help="Display metadata in real time. \n"
         "**WARNING:** This is still an **EXPERIMENTAL** feature and is **UNSTABLE**.",
     )(fn)
+
     return fn
 
 

miniscope_io/cli/update.py

@@ -28,8 +28,8 @@
     "-k",
     "--key",
     required=False,
-    type=click.Choice(["LED", "GAIN", "ROI_X", "ROI_Y"]),
-    help="key to update. Cannot be used with --restart.",
+    type=click.Choice(["LED", "GAIN", "ROI_X", "ROI_Y", "SUBSAMPLE"]),
+    help="key to update.",
 )
 @click.option(
     "-v",
@@ -52,16 +52,13 @@ def update(port: str, key: str, value: int, device_id: int, batch: str) -> None:
     """
     Update device configuration.
     """
-
     # Check mutual exclusivity
-    if (key and not value) or (value and not key):
+    if (key and value is None) or (value and not key):
         raise click.UsageError("Both --key and --value are required if one is specified.")
 
     if batch and (key or value):
-        raise click.UsageError(
-            "Options --key/--value and --restart" " and --batch are mutually exclusive."
-        )
-    if key and value:
+        raise click.UsageError("Options --key/--value and --batch are mutually exclusive.")
+    if key and value is not None:
         device_update(port=port, key=key, value=value, device_id=device_id)
     elif batch:
         with open(batch) as f:
@@ -91,7 +88,7 @@ def update(port: str, key: str, value: int, device_id: int, batch: str) -> None:
     "--reboot",
     is_flag=True,
     type=bool,
-    help="Restart the device. Cannot be used with --key or --value.",
+    help="Restart the device.",
 )
 def device(port: str, device_id: int, reboot: bool) -> None:
     """

miniscope_io/device_update.py

@@ -11,13 +11,13 @@
 from miniscope_io.logging import init_logger
 from miniscope_io.models.devupdate import DevUpdateCommand, UpdateCommandDefinitions
 
-logger = init_logger(name="device_update", level="INFO")
+logger = init_logger(name="device_update")
 FTDI_VENDOR_ID = 0x0403
 FTDI_PRODUCT_ID = 0x6001
 
 
 def device_update(
-    target: str,
+    key: str,
     value: int,
     device_id: int,
     port: Optional[str] = None,
@@ -28,8 +28,8 @@ def device_update(
     Args:
         device_id: ID of the device. 0 will update all devices.
         port: Serial port to which the device is connected.
-        target: What to update on the device (e.g., LED, GAIN).
-        value: Value to which the target should be updated.
+        key: What to update on the device (e.g., LED, GAIN).
+        value: Value to which the key should be updated.
 
     Returns:
         None
@@ -47,8 +47,8 @@ def device_update(
             port = ftdi_port_list[0]
             logger.info(f"Using port {port}")
 
-    command = DevUpdateCommand(device_id=device_id, port=port, target=target, value=value)
-    logger.info(f"Updating {target} to {value} on port {port}")
+    command = DevUpdateCommand(device_id=device_id, port=port, key=key, value=value)
+    logger.info(f"Updating {key} to {value} on port {port}")
 
     try:
         serial_port = serial.Serial(port=command.port, baudrate=2400, timeout=5, stopbits=2)
@@ -63,9 +63,9 @@ def device_update(
         logger.debug(f"Command: {format(id_command, '08b')}; Device ID: {command.device_id}")
         time.sleep(0.1)
 
-        target_command = (command.target.value + UpdateCommandDefinitions.target_header) & 0xFF
-        serial_port.write(target_command.to_bytes(1, "big"))
-        logger.debug(f"Command: {format(target_command, '08b')}; Target: {command.target.name}")
+        key_command = (command.key.value + UpdateCommandDefinitions.key_header) & 0xFF
+        serial_port.write(key_command.to_bytes(1, "big"))
+        logger.debug(f"Command: {format(key_command, '08b')}; Key: {command.key.name}")
         time.sleep(0.1)
 
         value_LSB_command = (

miniscope_io/devices/mocks.py

@@ -56,11 +56,11 @@ def __init__(self, serial_id: str = ""):
         with open(self.DATA_FILE, "rb") as dfile:
             self._buffer = bytearray(dfile.read())
 
-    def uploadBit(self, bit_file: str) -> None:
+    def upload_bit(self, bit_file: str) -> None:
         assert Path(bit_file).exists()
         self.bit_file = Path(bit_file)
 
-    def readData(self, length: int, addr: int = 0xA0, blockSize: int = 16) -> bytearray:
+    def read_data(self, length: int, addr: int = 0xA0, blockSize: int = 16) -> bytearray:
         if self._buffer_position >= len(self._buffer):
             # Error if called after we have returned the last data
             raise EndOfRecordingException("End of sample buffer")
@@ -70,5 +70,5 @@ def readData(self, length: int, addr: int = 0xA0, blockSize: int = 16) -> bytear
         self._buffer_position = end_pos
         return data
 
-    def setWire(self, addr: int, val: int) -> None:
+    def set_wire(self, addr: int, val: int) -> None:
         self._wires[addr] = val

miniscope_io/devices/opalkelly.py

@@ -33,7 +33,7 @@ def __init__(self, serial_id: str = ""):
         if ret == self.NoError:
             self.logger.info(f"Connected to {self.info.productName}")
 
-    def uploadBit(self, bit_file: str) -> None:
+    def upload_bit(self, bit_file: str) -> None:
         """
         Upload a configuration bitfile to the FPGA
 
@@ -51,7 +51,7 @@ def uploadBit(self, bit_file: str) -> None:
         )
         ret = self.ResetFPGA()
 
-    def readData(self, length: int, addr: int = 0xA0, blockSize: int = 16) -> bytearray:
+    def read_data(self, length: int, addr: int = 0xA0, blockSize: int = 16) -> bytearray:
         """
         Read a buffer's worth of data
 
@@ -73,7 +73,7 @@ def readData(self, length: int, addr: int = 0xA0, blockSize: int = 16) -> bytear
             self.logger.warning(f"Only {ret} bytes read")
         return buf
 
-    def setWire(self, addr: int, val: int) -> None:
+    def set_wire(self, addr: int, val: int) -> None:
         """
         .. todo::
 

miniscope_io/models/devupdate.py

@@ -19,29 +19,32 @@ class UpdateCommandDefinitions:
     Definitions of Bit masks and headers for remote update commands.
     """
 
-    # Header to indicate target/value.
+    # Header to indicate key/value.
     # It probably won't be used in other places so defined here.
     id_header = 0b00000000
-    target_header = 0b11000000
+    key_header = 0b11000000
     LSB_header = 0b01000000
     MSB_header = 0b10000000
     LSB_value_mask = 0b000000111111  # value below 12-bit
     MSB_value_mask = 0b111111000000  # value below 12-bit
     reset_byte = 0b11111111
 
 
-class UpdateTarget(int, Enum):
+class UpdateKey(int, Enum):
     """
-    Targets to update. Needs to be under 6-bit.
+    Keys to update. Needs to be under 6-bit.
     """
 
     LED = 0
     GAIN = 1
     ROI_X = 2
     ROI_Y = 3
+    SUBSAMPLE = 4
+    """
     ROI_WIDTH = 4  # not implemented
     ROI_HEIGHT = 5  # not implemented
     EWL = 6  # not implemented
+    """
     DEVICE = 50  # for device commands
 
 
@@ -52,31 +55,34 @@ class DevUpdateCommand(BaseModel):
 
     device_id: int
     port: str
-    target: UpdateTarget
+    key: UpdateKey
     value: int
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
     @model_validator(mode="after")
     def validate_values(cls, values: dict) -> dict:
         """
-        Validate values based on target.
+        Validate values based on key.
         """
-        target = values.target
+        key = values.key
         value = values.value
 
-        if target == UpdateTarget.LED:
+        if key == UpdateKey.LED:
             assert 0 <= value <= 100, "For LED, value must be between 0 and 100"
-        elif target == UpdateTarget.GAIN:
+        elif key == UpdateKey.GAIN:
             assert value in [1, 2, 4], "For GAIN, value must be 1, 2, or 4"
-        elif target == UpdateTarget.DEVICE:
+        elif key == UpdateKey.DEVICE:
             assert value in [DeviceCommand.REBOOT.value], "For DEVICE, value must be in [200]"
-        elif target in UpdateTarget:
+        elif key == UpdateKey.SUBSAMPLE:
+            assert value in [0, 1], "For SUBSAMPLE, value must be in [0, 1]"
+        elif key in [UpdateKey.ROI_X, UpdateKey.ROI_Y]:
+            # validation not implemented
+            pass
+        elif key in UpdateKey:
             raise NotImplementedError()
         else:
-            raise ValueError(
-                f"{target} is not a valid update target," "need an instance of UpdateTarget"
-            )
+            raise ValueError(f"{key} is not a valid update key," "need an instance of UpdateKey")
         return values
 
     @field_validator("port")
@@ -101,24 +107,24 @@ def validate_port(cls, value: str) -> str:
             raise ValueError(f"Port {value} not found")
         return value
 
-    @field_validator("target", mode="before")
-    def validate_target(cls, value: str) -> UpdateTarget:
+    @field_validator("key", mode="before")
+    def validate_key(cls, value: str) -> UpdateKey:
         """
-        Validate and convert target string to UpdateTarget Enum type.
+        Validate and convert key string to UpdateKey Enum type.
 
         Args:
-            value (str): Target to validate.
+            value (str): Key to validate.
 
         Returns:
-            UpdateTarget: Validated target as UpdateTarget.
+            UpdateKey: Validated key as UpdateKey.
 
         Raises:
-            ValueError: If target not found.
+            ValueError: If key not found.
         """
         try:
-            return UpdateTarget[value]
+            return UpdateKey[value]
         except KeyError as e:
             raise ValueError(
-                f"Target {value} not found, must be a member of UpdateTarget:"
-                f" {list(UpdateTarget.__members__.keys())}."
+                f"Key {value} not found, must be a member of UpdateKey:"
+                f" {list(UpdateKey.__members__.keys())}."
             ) from e

miniscope_io/models/stream.py

@@ -146,6 +146,10 @@ class StreamDevRuntime(MiniscopeConfig):
         5,
         description="Buffer length for storing images in streamDaq",
     )
+    queue_put_timeout: int = Field(
+        5,
+        description="Timeout for putting data into the queue",
+    )
     plot: Optional[StreamPlotterConfig] = Field(
         StreamPlotterConfig(
             keys=["timestamp", "buffer_count", "frame_buffer_count"], update_ms=1000, history=500