From beec7d0983200ca96b5189457801e1c00580d717 Mon Sep 17 00:00:00 2001 From: sneakers-the-rat Date: Fri, 21 Jun 2024 03:51:42 -0700 Subject: [PATCH] lint!!! --- miniscope_io/data.py | 20 ++--- miniscope_io/device_update.py | 4 +- miniscope_io/devices/opalkelly.py | 8 +- miniscope_io/io.py | 40 +++------ miniscope_io/logging.py | 14 +-- miniscope_io/models/__init__.py | 4 + miniscope_io/models/config.py | 14 ++- miniscope_io/models/mixins.py | 1 + miniscope_io/models/sdcard.py | 11 ++- miniscope_io/models/stream.py | 56 +++++++++--- miniscope_io/plots/__init__.py | 3 + miniscope_io/plots/headers.py | 21 ++++- miniscope_io/stream_daq.py | 141 +++++++++++++----------------- miniscope_io/utils.py | 4 + pyproject.toml | 4 +- 15 files changed, 187 insertions(+), 158 deletions(-) diff --git a/miniscope_io/data.py b/miniscope_io/data.py index dd8cd5fb..7e0423dd 100644 --- a/miniscope_io/data.py +++ b/miniscope_io/data.py @@ -23,20 +23,14 @@ class Frame(BaseModel, arbitrary_types_allowed=True): @field_validator("headers") @classmethod - def frame_nums_must_be_equal( - cls, v: List[SDBufferHeader] - ) -> Optional[List[SDBufferHeader]]: + def frame_nums_must_be_equal(cls, v: List[SDBufferHeader]) -> Optional[List[SDBufferHeader]]: """ Each frame_number field in each header must be the same (they come from the same frame!) """ - if v is not None and not all( - [header.frame_num != v[0].frame_num for header in v] - ): - raise ValueError( - f"All frame numbers should be equal! Got f{[h.frame_num for h in v]}" - ) + if v is not None and not all([header.frame_num != v[0].frame_num for header in v]): + raise ValueError(f"All frame numbers should be equal! Got f{[h.frame_num for h in v]}") return v @property @@ -55,16 +49,12 @@ class Frames(BaseModel): frames: List[Frame] @overload - def flatten_headers( - self, as_dict: Literal[False] = False - ) -> List[SDBufferHeader]: ... + def flatten_headers(self, as_dict: Literal[False] = False) -> List[SDBufferHeader]: ... @overload def flatten_headers(self, as_dict: Literal[True] = True) -> List[dict]: ... - def flatten_headers( - self, as_dict: bool = False - ) -> Union[List[dict], List[SDBufferHeader]]: + def flatten_headers(self, as_dict: bool = False) -> Union[List[dict], List[SDBufferHeader]]: """ Return flat list of headers, not grouped by frame diff --git a/miniscope_io/device_update.py b/miniscope_io/device_update.py index 4c8d50fa..255de749 100644 --- a/miniscope_io/device_update.py +++ b/miniscope_io/device_update.py @@ -126,9 +126,7 @@ def updateDevice() -> None: # set up serial port try: - serial_port = serial.Serial( - port=comport, baudrate=baudrate, timeout=5, stopbits=1 - ) + serial_port = serial.Serial(port=comport, baudrate=baudrate, timeout=5, stopbits=1) except Exception as e: logger.exception(e) raise e diff --git a/miniscope_io/devices/opalkelly.py b/miniscope_io/devices/opalkelly.py index 6206c2c3..108abdad 100644 --- a/miniscope_io/devices/opalkelly.py +++ b/miniscope_io/devices/opalkelly.py @@ -45,13 +45,9 @@ def uploadBit(self, bit_file: str) -> None: if ret == self.NoError: self.logger.debug(f"Succesfully uploaded {bit_file}") else: - raise DeviceConfigurationError( - f"Configuration of {self.info.productName} failed" - ) + raise DeviceConfigurationError(f"Configuration of {self.info.productName} failed") self.logger.debug( - "FrontPanel {} supported".format( - "is" if self.IsFrontPanelEnabled() else "not" - ) + "FrontPanel {} supported".format("is" if self.IsFrontPanelEnabled() else "not") ) ret = self.ResetFPGA() diff --git a/miniscope_io/io.py b/miniscope_io/io.py index e67b9a86..fe788d2f 100644 --- a/miniscope_io/io.py +++ b/miniscope_io/io.py @@ -65,9 +65,7 @@ def config(self) -> SDConfig: if self._config is None: with open(self.drive, "rb") as sd: sd.seek(self.layout.sectors.config_pos, 0) - configSectorData = np.frombuffer( - sd.read(self.layout.sectors.size), dtype=np.uint32 - ) + configSectorData = np.frombuffer(sd.read(self.layout.sectors.size), dtype=np.uint32) self._config = SDConfig( **{ @@ -159,8 +157,7 @@ def frame_count(self) -> int: self._frame_count = int( np.ceil( - (self.config.n_buffers_recorded + self.config.n_buffers_dropped) - / len(headers) + (self.config.n_buffers_recorded + self.config.n_buffers_dropped) / len(headers) ) ) @@ -187,9 +184,7 @@ def __enter__(self) -> "SDCard": # init private attrs # create an empty frame to hold our data! - self._array = np.zeros( - (self.config.width * self.config.height, 1), dtype=np.uint8 - ) + self._array = np.zeros((self.config.width * self.config.height, 1), dtype=np.uint8) self._pixel_count = 0 self._last_buffer_n = 0 self._frame = 0 @@ -202,7 +197,7 @@ def __enter__(self) -> "SDCard": return self - def __exit__(self, exc_type, exc_value, traceback): + def __exit__(self, exc_type, exc_val, exc_tb): # noqa: ANN001 self._f.close() self._f = None self._frame = 0 @@ -224,9 +219,7 @@ def _read_data_header(self, sd: BinaryIO) -> SDBufferHeader: dataHeader = np.append( dataHeader, np.frombuffer( - sd.read( - (dataHeader[self.layout.buffer.length] - 1) * self.layout.word_size - ), + sd.read((dataHeader[self.layout.buffer.length] - 1) * self.layout.word_size), dtype=np.uint32, ), ) @@ -265,9 +258,7 @@ def _read_size(self, header: SDBufferHeader) -> int: them separate in case they are separable actions for now """ n_blocks = self._n_frame_blocks(header) - read_size = (n_blocks * self.layout.sectors.size) - ( - header.length * self.layout.word_size - ) + read_size = (n_blocks * self.layout.sectors.size) - (header.length * self.layout.word_size) return read_size def _read_buffer(self, sd: BinaryIO, header: SDBufferHeader) -> np.ndarray: @@ -340,18 +331,14 @@ def read(self, return_header: bool = False) -> Union[np.ndarray, Frame]: # blank, and thus have a value of 0 for the header size, and we # can't read 0 from the card. self._f.seek(last_position, 0) - raise EndOfRecordingException( - "Reached the end of the video!" - ) from None + raise EndOfRecordingException("Reached the end of the video!") from None else: raise e except IndexError as e: if "index 0 is out of bounds for axis 0 with size 0" in str(e): # end of file if we are reading from a disk image without any # additional space on disk - raise EndOfRecordingException( - "Reached the end of the video!" - ) from None + raise EndOfRecordingException("Reached the end of the video!") from None else: raise e @@ -387,7 +374,7 @@ def to_video( isColor: bool = False, force: bool = False, progress: bool = True, - ): + ) -> None: """ Save contents of SD card to video with opencv @@ -411,8 +398,7 @@ def to_video( path = Path(path) if path.exists() and not force: raise FileExistsError( - f"{str(path)} already exists, not overwriting. " - "Use force=True to overwrite." + f"{str(path)} already exists, not overwriting. " "Use force=True to overwrite." ) if progress: @@ -454,7 +440,7 @@ def to_video( # General Methods # -------------------------------------------------- - def skip(self): + def skip(self) -> None: """ Skip a frame @@ -502,9 +488,7 @@ def check_valid(self) -> bool: """ with open(self.drive, "rb") as sd: sd.seek(self.layout.sectors.header_pos, 0) - headerSectorData = np.frombuffer( - sd.read(self.layout.sectors.size), dtype=np.uint32 - ) + headerSectorData = np.frombuffer(sd.read(self.layout.sectors.size), dtype=np.uint32) valid = False if ( diff --git a/miniscope_io/logging.py b/miniscope_io/logging.py index 38e9f79a..3fba3eb3 100644 --- a/miniscope_io/logging.py +++ b/miniscope_io/logging.py @@ -1,3 +1,7 @@ +""" +Logging factory and handlers +""" + import logging from logging.handlers import RotatingFileHandler from pathlib import Path @@ -15,7 +19,7 @@ def init_logger( file_level: Optional[LOG_LEVELS] = None, log_file_n: Optional[int] = None, log_file_size: Optional[int] = None, -): +) -> logging.Logger: """ Make a logger. @@ -60,9 +64,7 @@ def init_logger( # Add handlers for stdout and file if log_dir is not False: - logger.addHandler( - _file_handler(name, file_level, log_dir, log_file_n, log_file_size) - ) + logger.addHandler(_file_handler(name, file_level, log_dir, log_file_n, log_file_size)) logger.addHandler(_rich_handler()) @@ -82,9 +84,7 @@ def _file_handler( file_handler = RotatingFileHandler( str(filename), mode="a", maxBytes=log_file_size, backupCount=log_file_n ) - file_formatter = logging.Formatter( - "[%(asctime)s] %(levelname)s [%(name)s]: %(message)s" - ) + file_formatter = logging.Formatter("[%(asctime)s] %(levelname)s [%(name)s]: %(message)s") file_handler.setLevel(file_level) file_handler.setFormatter(file_formatter) return file_handler diff --git a/miniscope_io/models/__init__.py b/miniscope_io/models/__init__.py index fc1f5b38..ed520844 100644 --- a/miniscope_io/models/__init__.py +++ b/miniscope_io/models/__init__.py @@ -1,3 +1,7 @@ +""" +Data models :) +""" + from miniscope_io.models.models import Container, MiniscopeConfig, MiniscopeIOModel __all__ = [ diff --git a/miniscope_io/models/config.py b/miniscope_io/models/config.py index 9c817ed0..51f096e8 100644 --- a/miniscope_io/models/config.py +++ b/miniscope_io/models/config.py @@ -1,3 +1,7 @@ +""" +Module-global configuration models +""" + from pathlib import Path from typing import Literal, Optional @@ -38,7 +42,10 @@ class LogConfig(MiniscopeIOModel): @field_validator("level", "level_file", "level_stdout", mode="before") @classmethod - def uppercase_levels(cls, value: Optional[str] = None): + def uppercase_levels(cls, value: Optional[str] = None) -> Optional[str]: + """ + Ensure log level strings are uppercased + """ if value is not None: value = value.upper() return value @@ -74,7 +81,8 @@ class Config(BaseSettings): base_dir: Path = Field( _default_basedir, - description="Base directory to store configuration and other temporary files, other paths are relative to this by default", + description="Base directory to store configuration and other temporary files, " + "other paths are relative to this by default", ) log_dir: Path = Field(Path("logs"), description="Location to store logs") logs: LogConfig = Field(LogConfig(), description="Additional settings for logs") @@ -82,6 +90,7 @@ class Config(BaseSettings): @field_validator("base_dir", mode="before") @classmethod def folder_exists(cls, v: Path) -> Path: + """Ensure base_dir exists, make it otherwise""" v = Path(v) v.mkdir(exist_ok=True, parents=True) @@ -90,6 +99,7 @@ def folder_exists(cls, v: Path) -> Path: @model_validator(mode="after") def paths_relative_to_basedir(self) -> "Config": + """If relative paths are given, make them absolute relative to ``base_dir``""" paths = ("log_dir",) for path_name in paths: path = getattr(self, path_name) # type: Path diff --git a/miniscope_io/models/mixins.py b/miniscope_io/models/mixins.py index a88d7280..18723ce3 100644 --- a/miniscope_io/models/mixins.py +++ b/miniscope_io/models/mixins.py @@ -19,6 +19,7 @@ class YAMLMixin: @classmethod def from_yaml(cls: Type[T], file_path: Union[str, Path]) -> T: + """Instantiate this class by passing the contents of a yaml file as kwargs""" with open(file_path) as file: config_data = yaml.safe_load(file) return cls(**config_data) diff --git a/miniscope_io/models/sdcard.py b/miniscope_io/models/sdcard.py index 0af87195..1a00bdf8 100644 --- a/miniscope_io/models/sdcard.py +++ b/miniscope_io/models/sdcard.py @@ -117,12 +117,14 @@ class SDLayout(MiniscopeConfig): write_key2: int = 0x0D7CBA17 write_key3: int = 0x0D7CBA17 """ - These don't seem to actually be used in the existing reading/writing code, but we will leave them here for continuity's sake :) + These don't seem to actually be used in the existing reading/writing code, + but we will leave them here for continuity's sake :) """ word_size: int = 4 """ - I'm actually not sure what this is, but 4 is hardcoded a few times in the existing notebook and it - appears to be used as a word size when reading from the SD card. + I'm actually not sure what this is, but 4 is hardcoded a few times in the + existing notebook and it appears to be used as a word size when + reading from the SD card. """ header: SDHeaderPositions = SDHeaderPositions() @@ -140,7 +142,8 @@ class SDConfig(MiniscopeConfig): """ The configuration of a recording taken on this SD card. - Read from the locations given in :class:`.ConfigPositions` for an SD card with a given :class:`.SDLayout` + Read from the locations given in :class:`.ConfigPositions` + for an SD card with a given :class:`.SDLayout` """ width: int diff --git a/miniscope_io/models/stream.py b/miniscope_io/models/stream.py index 00041ce4..24b71de6 100644 --- a/miniscope_io/models/stream.py +++ b/miniscope_io/models/stream.py @@ -15,6 +15,11 @@ class StreamBufferHeaderFormat(BufferHeaderFormat): + """ + Refinements of :class:`.BufferHeaderFormat` for + :class:`~miniscope_io.stream_daq.StreamDaq` + """ + pixel_count: Range @@ -26,14 +31,24 @@ class StreamDaqConfig(MiniscopeConfig, YAMLMixin): Parameters ---------- mode: str - Operation modes. Maybe having DAQ (normal operation), RAW_RECORD (for getting testdata), and RAW_REPLAY (for plugging in recorded data for testing), BER_MEASURE, makes sense. Details TBD. + Operation modes. Maybe having + + - DAQ (normal operation), + - RAW_RECORD (for getting testdata), and + - RAW_REPLAY (for plugging in recorded data for testing), + + makes sense. Details TBD. device: str Interface hardware used for receiving data. Current options are "OK" (Opal Kelly XEM 7310) and "UART" (generic UART-USB converters). Only "OK" is supported at the moment. bitstream: str, optional Required when device is "OK". - The configuration bitstream file to upload to the Opal Kelly board. This uploads a Manchester decoder HDL and different bitstream files are required to configure different data rates and bit polarity. This is a binary file synthesized using Vivado, and details for generating this file will be provided in later updates. + The configuration bitstream file to upload to the Opal Kelly board. + This uploads a Manchester decoder HDL and different bitstream files are required + to configure different data rates and bit polarity. + This is a binary file synthesized using Vivado, + and details for generating this file will be provided in later updates. port: str, optional Required when device is "UART". COM port connected to the UART-USB converter. @@ -47,12 +62,16 @@ class StreamDaqConfig(MiniscopeConfig, YAMLMixin): fs: int Framerate of acquired stream preamble: str - 32-bit preamble used to locate the start of each buffer. The header and image data follows this preamble. + 32-bit preamble used to locate the start of each buffer. + The header and image data follows this preamble. This is used as a hex but imported as a string because yaml doesn't support hex format. header_len : int, optional Length of header in bits. (For 32-bit words, 32 * number of words) - This is useful when not all the variable/words in the header are defined in :class:`.MetadataHeaderFormat`. - The user is responsible to ensure that `header_len` is larger than the largest bit position defined in :class:`.MetadataHeaderFormat` otherwise unexpected behavior might occur. + This is useful when not all the variable/words in the header are defined in + :class:`.MetadataHeaderFormat`. + The user is responsible to ensure that `header_len` is larger than the largest bit + position defined in :class:`.MetadataHeaderFormat` + otherwise unexpected behavior might occur. pix_depth : int, optional Bit-depth of each pixel, by default 8. buffer_block_length: int @@ -69,15 +88,22 @@ class StreamDaqConfig(MiniscopeConfig, YAMLMixin): This isn't strictly required for data reconstruction but useful for debugging. LSB : bool, optional Whether the sourse is in "LSB" mode or not, by default True. - If `not LSB`, then the incoming bitstream is expected to be in Most Significant Bit first mode and data are transmitted in normal order. - If `LSB`, then the incoming bitstream is in the format that each 32-bit words are bit-wise reversed on its own. - Furthermore, the order of 32-bit words in the pixel data within the buffer is reversed (but the order of words in the header is preserved). - Note that this format does not correspond to the usual LSB-first convention and the parameter name is chosen for the lack of better words. + If `not LSB`, then the incoming bitstream is expected to be in Most Significant Bit first + mode and data are transmitted in normal order. + If `LSB`, then the incoming bitstream is in the format that each 32-bit words are + bit-wise reversed on its own. + Furthermore, the order of 32-bit words in the pixel data within the buffer is reversed + (but the order of words in the header is preserved). + Note that this format does not correspond to the usual LSB-first convention + and the parameter name is chosen for the lack of better words. show_video : bool, optional Whether the video is showed in "real-time", by default True. ..todo:: - Takuya - double-check the definitions around blocks and buffers in the firmware and add description. + + Takuya - double-check the definitions around blocks and buffers in the + firmware and add description. + """ mode: Literal["DAQ", "RAW_RECORD", "RAW_REPLAY"] = "DAQ" @@ -99,6 +125,16 @@ class StreamDaqConfig(MiniscopeConfig, YAMLMixin): @field_validator("preamble", mode="before") def preamble_to_bytes(cls, value: Union[str, bytes, int]) -> bytes: + """ + Cast ``preamble`` to bytes. + + Args: + value (str, bytes, int): Recast from `str` (in yaml like ``preamble: "0x12345"`` ) + or `int` (in yaml like `preamble: 0x12345` + + Returns: + bytes + """ if isinstance(value, str): return bytes.fromhex(value) elif isinstance(value, int): diff --git a/miniscope_io/plots/__init__.py b/miniscope_io/plots/__init__.py index e69de29b..7c620aaa 100644 --- a/miniscope_io/plots/__init__.py +++ b/miniscope_io/plots/__init__.py @@ -0,0 +1,3 @@ +""" +Convenience plotting functions +""" diff --git a/miniscope_io/plots/headers.py b/miniscope_io/plots/headers.py index 0539338e..5a6636a1 100644 --- a/miniscope_io/plots/headers.py +++ b/miniscope_io/plots/headers.py @@ -1,8 +1,15 @@ +""" +Plot headers from :class:`.SDCard` +""" + import matplotlib.pyplot as plt import pandas as pd def buffer_count(headers: pd.DataFrame, ax: plt.Axes) -> plt.Axes: + """ + Plot number of buffers by time + """ cols = ("write_buffer_count", "dropped_buffer_count", "buffer_count") labels = ("Write Buffer", "Dropped Buffer", "Total Buffer") for col, label in zip(cols, labels): @@ -14,6 +21,9 @@ def buffer_count(headers: pd.DataFrame, ax: plt.Axes) -> plt.Axes: def dropped_buffers(headers: pd.DataFrame, ax: plt.Axes) -> plt.Axes: + """ + Plot number of buffers by time + """ ax.plot(headers["dropped_buffer_count"], label="Dropped buffers") ax.legend() ax.set_xlabel("Buffer index") @@ -21,6 +31,9 @@ def dropped_buffers(headers: pd.DataFrame, ax: plt.Axes) -> plt.Axes: def timestamps(headers: pd.DataFrame, ax: plt.Axes) -> plt.Axes: + """ + Plot frame number against time + """ frames = headers["frame_num"].max() - headers["frame_num"].min() seconds = headers["timestamp"].max() - headers["timestamp"].min() fps = frames / seconds @@ -33,6 +46,9 @@ def timestamps(headers: pd.DataFrame, ax: plt.Axes) -> plt.Axes: def battery_voltage(headers: pd.DataFrame, ax: plt.Axes) -> plt.Axes: + """ + Plot battery voltage against time + """ ax.plot(headers["timestamp"], headers["battery_voltage"], label="Battery voltage") ax.legend() ax.set_xlabel("Time [s]") @@ -50,10 +66,7 @@ def plot_headers(headers: pd.DataFrame) -> (plt.Figure, plt.Axes): Arguments: headers (:class:`pandas.DataFrame`): headers to plot """ - if "battery_voltage" in headers.columns: - subplots = 4 - else: - subplots = 3 + subplots = 4 if "battery_voltage" in headers.columns else 3 fig, ax = plt.subplots(1, subplots) diff --git a/miniscope_io/stream_daq.py b/miniscope_io/stream_daq.py index 099dd8e4..487f3dbf 100644 --- a/miniscope_io/stream_daq.py +++ b/miniscope_io/stream_daq.py @@ -1,3 +1,7 @@ +""" +DAQ For use with FPGA and Uart streaming video sources. +""" + import argparse import multiprocessing import os @@ -27,7 +31,9 @@ except (ImportError, ModuleNotFoundError): module_logger = init_logger("streamDaq") module_logger.warning( - "Could not import OpalKelly driver, you can't read from FPGA!\nCheck out Opal Kelly's website for install info\nhttps://docs.opalkelly.com/fpsdk/getting-started/" + "Could not import OpalKelly driver, you can't read from FPGA!\n" + "Check out Opal Kelly's website for install info\n" + "https://docs.opalkelly.com/fpsdk/getting-started/" ) # Parsers for daq inputs @@ -39,7 +45,8 @@ class StreamDaq: """ A combined class for configuring and reading frames from a UART and FPGA source. Supported devices and required inputs are described in StreamDaqConfig model documentation. - This function's entry point is the main function, which should be used from the stream_image_capture command installed with the package. + This function's entry point is the main function, which should be used from the + stream_image_capture command installed with the package. Example configuration yaml files are stored in /miniscope-io/config/. Examples @@ -48,12 +55,13 @@ class StreamDaq: Connected to XEM7310-A75 Succesfully uploaded /miniscope-io/miniscope_io/devices/selected_bitfile.bit FrontPanel is supported - [24-06-11T01:40:45] INFO [miniscope_io.streamDaq.frame] frame: 1570, bits lost: 0 stream_daq.py:524 - [24-06-11T01:40:46] INFO [miniscope_io.streamDaq.frame] frame: 1571, bits lost: 0 stream_daq.py:524 .. todo:: + Example section: add the terminal output when running the script - Phil/Takuya - docstrings for stream daq: what devices these correspond to, how to configure them, usage examples, tests + Phil/Takuya - docstrings for stream daq: what devices these correspond to, + how to configure them, usage examples, tests + """ def __init__( @@ -71,7 +79,8 @@ def __init__( DAQ configurations imported from the input yaml file. Examples and required properties can be found in /miniscope-io/config/example.yml header_fmt : MetadataHeaderFormat, optional - Header format used to parse information from buffer header, by default `MetadataHeaderFormat()`. + Header format used to parse information from buffer header, + by default `MetadataHeaderFormat()`. """ self.logger = init_logger("streamDaq") self.config = config @@ -148,7 +157,7 @@ def _parse_header( def _uart_recv( self, serial_buffer_queue: multiprocessing.Queue, comport: str, baudrate: int - ): + ) -> None: """ Receive buffers and push into serial_buffer_queue. Currently not supported. @@ -165,9 +174,7 @@ def _uart_recv( pre_bytes = bytes(bytearray(self.preamble.tobytes())[::-1]) # set up serial port - serial_port = serial.Serial( - port=comport, baudrate=baudrate, timeout=5, stopbits=1 - ) + serial_port = serial.Serial(port=comport, baudrate=baudrate, timeout=5, stopbits=1) self.logger.info("Serial port open: " + str(serial_port.name)) # Throw away the first buffer because it won't fully come in @@ -197,7 +204,8 @@ def _fpga_recv( The bits data are read in fixed chunks defined by `read_length`. Then we concatenate the chunks and try to look for `self.preamble` in the data. - The data between every pair of `self.preamble` is considered to be a single buffer and stored in `serial_buffer_queue`. + The data between every pair of `self.preamble` is considered to be a single buffer and + stored in `serial_buffer_queue`. Parameters ---------- @@ -205,7 +213,8 @@ def _fpga_recv( The queue holding the buffer data. read_length : int, optional Length of data to read in chunks (in number of bytes), by default None. - If `None`, an optimal length is estimated so that it roughly covers a single buffer and is an integer multiple of 16 bytes (as recommended by OpalKelly). + If `None`, an optimal length is estimated so that it roughly covers a single buffer + and is an integer multiple of 16 bytes (as recommended by OpalKelly). pre_first : bool, optional Whether preamble/header is returned at the beginning of each buffer, by default True. @@ -220,23 +229,16 @@ def _fpga_recv( ) # determine length if read_length is None: - read_length = ( - int(max(self.buffer_npix) * self.config.pix_depth / 8 / 16) * 16 - ) + read_length = int(max(self.buffer_npix) * self.config.pix_depth / 8 / 16) * 16 # set up fpga devices BIT_FILE = self.config.bitstream if not BIT_FILE.exists(): - raise RuntimeError( - f"Configured to use bitfile at {BIT_FILE} but no such file exists" - ) + raise RuntimeError(f"Configured to use bitfile at {BIT_FILE} but no such file exists") # set up fpga devices # FIXME: when multiprocessing bug resolved, remove this and just mock in tests - if os.environ.get("PYTEST_CURRENT_TEST") is not None: - dev = okDevMock() - else: - dev = okDev() + dev = okDevMock() if os.environ.get("PYTEST_CURRENT_TEST") is not None else okDev() dev.uploadBit(str(BIT_FILE)) dev.setWire(0x00, 0b0010) @@ -258,7 +260,8 @@ def _fpga_recv( break if self.config.mode == "RAW_RECORD": - fpga_raw_path = "data/fpga_raw.bin" # Not sure where to define this because we don't want to overwrite. + # Not sure where to define this because we don't want to overwrite. + fpga_raw_path = "data/fpga_raw.bin" with open(fpga_raw_path, "wb") as file: file.write(buf) self.terminate.value = True @@ -267,9 +270,9 @@ def _fpga_recv( pre_pos = list(cur_buffer.findall(pre)) for buf_start, buf_stop in zip(pre_pos[:-1], pre_pos[1:]): if not pre_first: - buf_start, buf_stop = buf_start + len( + buf_start, buf_stop = buf_start + len(self.preamble), buf_stop + len( self.preamble - ), buf_stop + len(self.preamble) + ) serial_buffer_queue.put(cur_buffer[buf_start:buf_stop].tobytes()) if pre_pos: cur_buffer = cur_buffer[pre_pos[-1] :] @@ -278,12 +281,14 @@ def _buffer_to_frame( self, serial_buffer_queue: multiprocessing.Queue, frame_buffer_queue: multiprocessing.Queue, - ): + ) -> None: """ Group buffers together to make frames. - Pull out buffers in `serial_buffer_queue`, then get frame and buffer index by parsing headers in the buffer. - The buffers belonging to the same frame are put in the same list at corresponding buffer index. + Pull out buffers in `serial_buffer_queue`, then get frame and buffer index by + parsing headers in the buffer. + The buffers belonging to the same frame are put in the same list at + corresponding buffer index. The lists representing each frame are then put into `frame_buffer_queue`. Parameters @@ -301,12 +306,8 @@ def _buffer_to_frame( frame_buffer = [None] * self.nbuffer_per_fm while 1: - if ( - serial_buffer_queue.qsize() > 0 - ): # Higher is safe but lower should be faster. - serial_buffer = Bits( - serial_buffer_queue.get() - ) # grab one buffer from queue + if serial_buffer_queue.qsize() > 0: # Higher is safe but lower should be faster. + serial_buffer = Bits(serial_buffer_queue.get()) # grab one buffer from queue header_data, serial_buffer = self._parse_header(serial_buffer) @@ -343,9 +344,7 @@ def _buffer_to_frame( ): cur_fm_buffer_index = header_data.frame_buffer_count frame_buffer[cur_fm_buffer_index] = serial_buffer.tobytes() - locallogs.debug( - "----buffer #" + str(cur_fm_buffer_index) + " stored" - ) + locallogs.debug("----buffer #" + str(cur_fm_buffer_index) + " stored") # if lost frame from buffer -> reset index else: @@ -355,13 +354,18 @@ def _format_frame( self, frame_buffer_queue: multiprocessing.Queue, imagearray: multiprocessing.Queue, - ): + ) -> None: """ Construct frame from grouped buffers. - Each frame data is concatenated from a list of buffers in `frame_buffer_queue` according to `buffer_npix`. - If there is any mismatch between the expected length of each buffer (defined by `buffer_npix`) and the actual length, then the buffer is either truncated or zero-padded at the end to make the length appropriate, and a warning is thrown. - Finally, the concatenated buffer data are converted into a 1d numpy array with uint8 dtype and put into `imagearray` queue. + Each frame data is concatenated from a list of buffers in `frame_buffer_queue` + according to `buffer_npix`. + If there is any mismatch between the expected length of each buffer + (defined by `buffer_npix`) and the actual length, then the buffer is either + truncated or zero-padded at the end to make the length appropriate, + and a warning is thrown. + Finally, the concatenated buffer data are converted into a 1d numpy array with + uint8 dtype and put into `imagearray` queue. Parameters ---------- @@ -386,9 +390,7 @@ def _format_frame( Bits(frame_data[i]), truncate="header" ) else: - frame_data[i] = Bits( - int=0, length=npix_expected * self.config.pix_depth - ) + frame_data[i] = Bits(int=0, length=npix_expected * self.config.pix_depth) nbit_lost += npix_expected continue npix_header = header_data.pixel_count @@ -397,7 +399,11 @@ def _format_frame( if npix_actual != npix_expected: if i < len(self.buffer_npix) - 1: locallogs.warning( - f"Pixel count inconsistent for frame {header_data.frame_num} buffer {header_data.frame_buffer_count}. Expected: {npix_expected}, Header: {npix_header}, Actual: {npix_actual}" + f"Pixel count inconsistent for frame {header_data.frame_num} " + f"buffer {header_data.frame_buffer_count}. " + f"Expected: {npix_expected}, " + f"Header: {npix_header}, " + f"Actual: {npix_actual}" ) nbit_expected = npix_expected * self.config.pix_depth if len(fm_dat) > nbit_expected: @@ -414,9 +420,7 @@ def _format_frame( pixel_vector = pixel_vector + d assert len(pixel_vector) == ( - self.config.frame_height - * self.config.frame_width - * self.config.pix_depth + self.config.frame_height * self.config.frame_width * self.config.pix_depth ) if self.config.LSB: @@ -431,11 +435,9 @@ def _format_frame( imagearray.put(img) if header_data is not None: - locallogs.info( - f"frame: {header_data.frame_num}, bits lost: {nbit_lost}" - ) + locallogs.info(f"frame: {header_data.frame_num}, bits lost: {nbit_lost}") - def init_video(self, path: Path, fourcc: str = "Y800", **kwargs) -> cv2.VideoWriter: + def init_video(self, path: Path, fourcc: str = "Y800", **kwargs: dict) -> cv2.VideoWriter: """ Create a parameterized video writer @@ -459,7 +461,7 @@ def capture( read_length: Optional[int] = None, video: Optional[Path] = None, video_kwargs: Optional[dict] = None, - ): + ) -> None: """ Entry point to start frame capture. @@ -468,7 +470,8 @@ def capture( source : Literal[uart, fpga] Device source. read_length : Optional[int], optional - Passed to :function:`~miniscope_io.stream_daq.stream_daq._fpga_recv` when `source == "fpga"`, by default None. + Passed to :function:`~miniscope_io.stream_daq.stream_daq._fpga_recv` when + `source == "fpga"`, by default None. video: Path, optional If present, a path to an output video file video_options: dict, optional @@ -485,13 +488,10 @@ def capture( serial_buffer_queue = queue_manager.Queue( 10 ) # b'\x00' # hand over single buffer: uart_recv() -> buffer_to_frame() - frame_buffer_queue = queue_manager.Queue( - 5 - ) # [b'\x00', b'\x00', b'\x00', b'\x00', b'\x00'] # hand over a frame (five buffers): buffer_to_frame() + frame_buffer_queue = queue_manager.Queue(5) # [b'\x00', b'\x00', b'\x00', b'\x00', b'\x00'] + # hand over a frame (five buffers): buffer_to_frame() imagearray = queue_manager.Queue(5) - imagearray.put( - np.zeros(int(self.config.frame_width * self.config.frame_height), np.uint8) - ) + imagearray.put(np.zeros(int(self.config.frame_width * self.config.frame_height), np.uint8)) if source == "uart": self.logger.debug("Starting uart capture process") @@ -524,10 +524,6 @@ def capture( else: writer = None - def check_termination_flag(termination_flag): - input("Press enter to exit the process.") - termination_flag.value = True - p_buffer_to_frame = multiprocessing.Process( target=self._buffer_to_frame, args=( @@ -602,7 +598,7 @@ def check_termination_flag(termination_flag): break # watchdog process daemon gets [Terminated] -def main(): +def main() -> None: # noqa: D103 args = daqParser.parse_args() daqConfig = StreamDaqConfig.from_yaml(args.config) @@ -610,24 +606,13 @@ def main(): daq_inst = StreamDaq(config=daqConfig) if daqConfig.device == "UART": - try: - comport = daqConfig.port - except (ValueError, IndexError) as e: - print(e) - sys.exit(1) - - try: - baudrate = daqConfig.baudrate - except (ValueError, IndexError) as e: - print(e) - sys.exit(1) - # daq_inst.capture(source="uart", comport=comport, baudrate=baudrate) daq_inst.capture(source="uart") if daqConfig.device == "OK": if not HAVE_OK: raise ImportError( - "Requested Opal Kelly DAQ, but okDAQ could not be imported, got exception: {ok_error}" + f"Requested Opal Kelly DAQ, but okDAQ could not be imported, got exception: " + f"{ok_error}" ) daq_inst.capture(source="fpga") diff --git a/miniscope_io/utils.py b/miniscope_io/utils.py index 3e757467..bd0850e5 100644 --- a/miniscope_io/utils.py +++ b/miniscope_io/utils.py @@ -1,3 +1,7 @@ +""" +The junk drawer my dogs +""" + import hashlib from pathlib import Path from typing import TYPE_CHECKING, Union diff --git a/pyproject.toml b/pyproject.toml index 4c08b0c2..fa71cee9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,7 @@ updateDevice = "miniscope_io.device_update:updateDevice" target-version = "py311" include = ["miniscope_io/**/*.py", "pyproject.toml"] exclude = ["tests", "miniscope_io/vendor"] +line-length = 100 [tool.ruff.lint] select = [ @@ -114,4 +115,5 @@ plugins = [ [tool.black] target-version = ['py38', 'py39', 'py310', 'py311', 'py312'] include = "miniscope_io/.*\\.py$" -extend-exclude = 'miniscope_io/vendor/.*' \ No newline at end of file +extend-exclude = 'miniscope_io/vendor/.*' +line-length = 100 \ No newline at end of file