From 1ca8add1b9271ba27ad159ebea61f172a4e0177e Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 22 Aug 2023 11:24:29 -0700 Subject: [PATCH 01/39] add pydocstyle to pre-commit hooks --- .pre-commit-config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 812dc14d..38d4bd0b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,3 +10,7 @@ repos: hooks: - id: black exclude: ^docs/ +- repo: https://github.com/pycqa/pydocstyle + rev: 6.3.0 + hooks: + - id: pydocstyle From c82ac6f8ab547f1ac81bd2c400d0e1fb637796df Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 22 Aug 2023 16:50:17 -0700 Subject: [PATCH 02/39] add docstrings to suite2p --- .../extractors/suite2p/__init__.py | 12 +++ .../suite2p/suite2psegmentationextractor.py | 96 ++++++++++++++++++- 2 files changed, 106 insertions(+), 2 deletions(-) diff --git a/src/roiextractors/extractors/suite2p/__init__.py b/src/roiextractors/extractors/suite2p/__init__.py index f3ba893f..fb8d97f5 100644 --- a/src/roiextractors/extractors/suite2p/__init__.py +++ b/src/roiextractors/extractors/suite2p/__init__.py @@ -1 +1,13 @@ +"""A segmentation extractor for Suite2p. + +Modules +------- +suite2psegmentationextractor + A segmentation extractor for Suite2p. + +Classes +------- +Suite2pSegmentationExtractor + A segmentation extractor for Suite2p. +""" from .suite2psegmentationextractor import Suite2pSegmentationExtractor diff --git a/src/roiextractors/extractors/suite2p/suite2psegmentationextractor.py b/src/roiextractors/extractors/suite2p/suite2psegmentationextractor.py index fd64ce54..cd9aa2a2 100644 --- a/src/roiextractors/extractors/suite2p/suite2psegmentationextractor.py +++ b/src/roiextractors/extractors/suite2p/suite2psegmentationextractor.py @@ -1,3 +1,10 @@ +"""A segmentation extractor for Suite2p. + +Classes +------- +Suite2pSegmentationExtractor + A segmentation extractor for Suite2p. +""" import shutil from pathlib import Path from typing import Optional @@ -11,6 +18,8 @@ class Suite2pSegmentationExtractor(SegmentationExtractor): + """A segmentation extractor for Suite2p.""" + extractor_name = "Suite2pSegmentationExtractor" installed = True # check at class level if installed or not is_writable = False @@ -24,8 +33,8 @@ def __init__( plane_no: IntType = 0, file_path: Optional[PathType] = None, ): - """ - Creating SegmentationExtractor object out of suite 2p data type. + """Create SegmentationExtractor object out of suite 2p data type. + Parameters ---------- folder_path: str or Path @@ -70,16 +79,43 @@ def __init__( self._image_mean = self._summary_image_read("meanImg") def _load_npy(self, filename, mmap_mode=None): + """Load a .npy file with specified filename. + + Parameters + ---------- + filename: str + The name of the .npy file to load. + mmap_mode: str + The mode to use for memory mapping. See numpy.load for details. + + Returns + ------- + The loaded .npy file. + """ file_path = self.folder_path / f"plane{self.plane_no}" / filename return np.load(file_path, mmap_mode=mmap_mode, allow_pickle=mmap_mode is None) def get_accepted_list(self): + """Return a list of accepted ROI ids.""" return list(np.where(self.iscell[:, 0] == 1)[0]) def get_rejected_list(self): + """Return a list of rejected ROI ids.""" return list(np.where(self.iscell[:, 0] == 0)[0]) def _summary_image_read(self, bstr="meanImg"): + """Read summary image from ops (settings) dict. + + Parameters + ---------- + bstr: str + The name of the summary image to read. + + Returns + ------- + img : numpy.ndarray | None + The summary image if bstr is in ops, else None. + """ img = None if bstr in self.ops: if bstr == "Vcorr" or bstr == "max_proj": @@ -94,9 +130,22 @@ def _summary_image_read(self, bstr="meanImg"): @property def roi_locations(self): + """Returns the center locations (x, y) of each ROI.""" return np.array([j["med"] for j in self.stat]).T.astype(int) def get_roi_image_masks(self, roi_ids=None): + """Get image masks for all ROIs specified by roi_ids. + + Parameters + ---------- + roi_ids: list + A list of ROI ids to get image masks for. If None, all ROIs are used. + + Returns + ------- + image_masks: numpy.ndarray + A 3D numpy array of image masks with shape (y, x, len(roi_ids)). + """ if roi_ids is None: roi_idx_ = range(self.get_num_rois()) else: @@ -110,6 +159,18 @@ def get_roi_image_masks(self, roi_ids=None): ) def get_roi_pixel_masks(self, roi_ids=None): + """Get pixel masks for all ROIs specified by roi_ids. + + Parameters + ---------- + roi_ids: list + A list of ROI ids to get pixel masks for. If None, all ROIs are used. + + Returns + ------- + pixel_masks: list + A list of pixel masks for each ROI. + """ pixel_mask = [] for i in range(self.get_num_rois()): pixel_mask.append( @@ -130,10 +191,41 @@ def get_roi_pixel_masks(self, roi_ids=None): return [pixel_mask[i] for i in roi_idx_] def get_image_size(self): + """Return the shape of the image (y, x).""" return [self.ops["Ly"], self.ops["Lx"]] @staticmethod def write_segmentation(segmentation_object: SegmentationExtractor, save_path: PathType, overwrite=True): + """Write a SegmentationExtractor to a folder specified by save_path. + + Parameters + ---------- + segmentation_object: SegmentationExtractor + The SegmentationExtractor object to be written. + save_path: str or Path + The folder path where to write the segmentation. + overwrite: bool + If True, overwrite the folder if it already exists. + + Raises + ------ + AssertionError + If save_path is not a folder. + FileExistsError + If the folder already exists and overwrite is False. + + Notes + ----- + The folder structure is as follows: + save_path + └── plane + ├── F.npy + ├── Fneu.npy + ├── spks.npy + ├── stat.npy + ├── iscell.npy + └── ops.npy + """ save_path = Path(save_path) assert not save_path.is_file(), "'save_path' must be a folder" From d1d99e72afd2408be38e9c7b73b02a5a1c6dda72 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Wed, 23 Aug 2023 09:59:25 -0700 Subject: [PATCH 03/39] add docstrings to hdf5 --- .../hdf5imagingextractor/__init__.py | 13 +++ .../hdf5imagingextractor.py | 84 +++++++++++++++++++ 2 files changed, 97 insertions(+) diff --git a/src/roiextractors/extractors/hdf5imagingextractor/__init__.py b/src/roiextractors/extractors/hdf5imagingextractor/__init__.py index 876ed4b4..ec01de58 100644 --- a/src/roiextractors/extractors/hdf5imagingextractor/__init__.py +++ b/src/roiextractors/extractors/hdf5imagingextractor/__init__.py @@ -1 +1,14 @@ +"""An imaging extractor for HDF5. + +Modules +------- +hdf5imagingextractor + An imaging extractor for HDF5. + + +Classes +------- +Hdf5ImagingExtractor + An imaging extractor for HDF5. +""" from .hdf5imagingextractor import Hdf5ImagingExtractor diff --git a/src/roiextractors/extractors/hdf5imagingextractor/hdf5imagingextractor.py b/src/roiextractors/extractors/hdf5imagingextractor/hdf5imagingextractor.py index 77e573f7..b9344f0a 100644 --- a/src/roiextractors/extractors/hdf5imagingextractor/hdf5imagingextractor.py +++ b/src/roiextractors/extractors/hdf5imagingextractor/hdf5imagingextractor.py @@ -1,3 +1,10 @@ +"""An imaging extractor for HDF5. + +Classes +------- +Hdf5ImagingExtractor + An imaging extractor for HDF5. +""" from pathlib import Path from typing import Optional, Tuple from warnings import warn @@ -22,6 +29,8 @@ class Hdf5ImagingExtractor(ImagingExtractor): + """An imaging extractor for HDF5.""" + extractor_name = "Hdf5Imaging" installed = HAVE_H5 # check at class level if installed or not is_writable = True @@ -37,6 +46,23 @@ def __init__( metadata: dict = None, channel_names: ArrayType = None, ): + """Create an ImagingExtractor from an HDF5 file. + + Parameters + ---------- + file_path : str or Path + Path to the HDF5 file. + mov_field : str, optional + Name of the dataset in the HDF5 file that contains the imaging data. The default is "mov". + sampling_frequency : float, optional + Sampling frequency of the video. The default is None. + start_time : float, optional + Start time of the video. The default is None. + metadata : dict, optional + Metadata dictionary. The default is None. + channel_names : array-like, optional + List of channel names. The default is None. + """ ImagingExtractor.__init__(self) self.filepath = Path(file_path) @@ -91,9 +117,24 @@ def __init__( } def __del__(self): + """Close the HDF5 file.""" self._file.close() def get_frames(self, frame_idxs: ArrayType, channel: Optional[int] = 0): + """Return frames from the video. + + Parameters + ---------- + frame_idxs : array-like + 2-element list of starting frame and ending frame (inclusive). + channel : int, optional + Channel index. The default is 0. + + Returns + ------- + numpy.ndarray + Array of frames. + """ # Fancy indexing is non performant for h5.py with long frame lists if frame_idxs is not None: slice_start = np.min(frame_idxs) @@ -109,21 +150,42 @@ def get_frames(self, frame_idxs: ArrayType, channel: Optional[int] = 0): return frames def get_video(self, start_frame=None, end_frame=None, channel: Optional[int] = 0) -> np.ndarray: + """Return the video frames. + + Parameters + ---------- + start_frame : int, optional + Starting frame. The default is None. + end_frame : int, optional + Ending frame. The default is None. + channel : int, optional + Channel index. The default is 0. + + Returns + ------- + numpy.ndarray + Array of frames. + """ return self._video.lazy_slice[start_frame:end_frame, :, :, channel].dsetread() def get_image_size(self) -> Tuple[int, int]: + """Return the size of the video (num_rows, num_cols).""" return (self._num_rows, self._num_cols) def get_num_frames(self): + """Return the number of frames in the video.""" return self._num_frames def get_sampling_frequency(self): + """Return the sampling frequency of the video.""" return self._sampling_frequency def get_channel_names(self): + """Return the channel names.""" return self._channel_names def get_num_channels(self): + """Return the number of channels.""" return self._num_channels @staticmethod @@ -134,6 +196,28 @@ def write_imaging( mov_field="mov", **kwargs, ): + """Write an imaging extractor to an HDF5 file. + + Parameters + ---------- + imaging : ImagingExtractor + The imaging extractor object to be saved. + save_path : str or Path + Path to save the file. + overwrite : bool, optional + If True, overwrite the file if it already exists. The default is False. + mov_field : str, optional + Name of the dataset in the HDF5 file that contains the imaging data. The default is "mov". + **kwargs : dict + Keyword arguments to be passed to the HDF5 file writer. + + Raises + ------ + AssertionError + If the file extension is not .h5 or .hdf5. + FileExistsError + If the file already exists and overwrite is False. + """ save_path = Path(save_path) assert save_path.suffix in [ ".h5", From 7ea3d82f0dc28a8f207488edadf2b294068d82ac Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Wed, 23 Aug 2023 10:35:58 -0700 Subject: [PATCH 04/39] add docstrings to sima --- .../extractors/simaextractor/__init__.py | 12 ++++++ .../simasegmentationextractor.py | 39 +++++++++++++++---- .../suite2p/suite2psegmentationextractor.py | 2 +- 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/src/roiextractors/extractors/simaextractor/__init__.py b/src/roiextractors/extractors/simaextractor/__init__.py index a42c6d42..a8c1581f 100644 --- a/src/roiextractors/extractors/simaextractor/__init__.py +++ b/src/roiextractors/extractors/simaextractor/__init__.py @@ -1 +1,13 @@ +"""A segmentation extractor for Sima. + +Modules +------- +simasegmentationextractor + A segmentation extractor for Sima. + +Classes +------- +SimaSegmentationExtractor + A segmentation extractor for Sima. +""" from .simasegmentationextractor import SimaSegmentationExtractor diff --git a/src/roiextractors/extractors/simaextractor/simasegmentationextractor.py b/src/roiextractors/extractors/simaextractor/simasegmentationextractor.py index e75cf71b..638fe7fc 100644 --- a/src/roiextractors/extractors/simaextractor/simasegmentationextractor.py +++ b/src/roiextractors/extractors/simaextractor/simasegmentationextractor.py @@ -1,3 +1,10 @@ +"""A segmentation extractor for Sima. + +Classes +------- +SimaSegmentationExtractor + A segmentation extractor for Sima. +""" import os import pickle import re @@ -18,10 +25,11 @@ class SimaSegmentationExtractor(SegmentationExtractor): - """ + """A segmentation extractor for Sima. + This class inherits from the SegmentationExtractor class, having all its functionality specifically applied to the dataset output from - the \'SIMA\' ROI segmentation method. + the 'SIMA' ROI segmentation method. """ extractor_name = "SimaSegmentation" @@ -32,10 +40,11 @@ class SimaSegmentationExtractor(SegmentationExtractor): installation_mesg = "To use the SimaSegmentationExtractor install sima and dill: \n\n pip install sima/dill\n\n" def __init__(self, file_path: PathType, sima_segmentation_label: str = "auto_ROIs"): - """ + """Create a SegmentationExtractor instance from a sima file. + Parameters ---------- - file_path: str + file_path: str or Path The location of the folder containing dataset.sima file and the raw image file(s) (tiff, h5, .zip) sima_segmentation_label: str @@ -55,13 +64,14 @@ def __init__(self, file_path: PathType, sima_segmentation_label: str = "auto_ROI @staticmethod def _convert_sima(old_pkl_loc): - """ + """Convert the sima file to python 3 pickle. + This function is used to convert python 2 pickles to python 3 pickles. - Forward compatibility of \'*.sima\' files containing .pkl dataset, rois, + Forward compatibility of '*.sima' files containing .pkl dataset, rois, sequences, signals, time_averages. Replaces the pickle file with a python 3 version with the same name. Saves - the old Py2 pickle as \'oldpicklename_p2.pkl\'' + the old Py2 pickle as 'oldpicklename_p2.pkl' Parameters ---------- @@ -96,11 +106,13 @@ def _convert_sima(old_pkl_loc): pickle.dump(loaded, outfile) def _file_extractor_read(self): + """Read the sima file and return the sima.ImagingDataset object.""" _img_dataset = sima.ImagingDataset.load(self.file_path) _img_dataset._savedir = self.file_path return _img_dataset def _image_mask_extractor_read(self): + """Read the image mask from the sima.ImagingDataset object (self._dataset_file).""" _sima_rois = self._dataset_file.ROIs if len(_sima_rois) > 1: if self.sima_segmentation_label in list(_sima_rois.keys()): @@ -116,6 +128,7 @@ def _image_mask_extractor_read(self): return np.array(image_masks_).T def _trace_extractor_read(self): + """Read the traces from the sima.ImagingDataset object (self._dataset_file).""" for channel_now in self._channel_names: for labels in self._dataset_file.signals(channel=channel_now): if labels: @@ -142,18 +155,28 @@ def _trace_extractor_read(self): return extracted_signals def _summary_image_read(self): + """Read the summary image from the sima.ImagingDataset object (self._dataset_file).""" summary_image = np.squeeze(self._dataset_file.time_averages[0]).T return np.array(summary_image).T def get_accepted_list(self): + """Return the list of accepted ROIs.""" return list(range(self.get_num_rois())) def get_rejected_list(self): + """Return the list of rejected ROIs.""" return [a for a in range(self.get_num_rois()) if a not in set(self.get_accepted_list())] @staticmethod def write_segmentation(segmentation_object, savepath): - raise NotImplementedError + """Write a segmentation object to a file. + + Notes + ----- + This function is not implemented for this extractor. + """ + raise NotImplementedError # TODO: implement write_segmentation def get_image_size(self): + """Return the size of the image (height, width).""" return self._image_masks.shape[0:2] diff --git a/src/roiextractors/extractors/suite2p/suite2psegmentationextractor.py b/src/roiextractors/extractors/suite2p/suite2psegmentationextractor.py index cd9aa2a2..f6535127 100644 --- a/src/roiextractors/extractors/suite2p/suite2psegmentationextractor.py +++ b/src/roiextractors/extractors/suite2p/suite2psegmentationextractor.py @@ -191,7 +191,7 @@ def get_roi_pixel_masks(self, roi_ids=None): return [pixel_mask[i] for i in roi_idx_] def get_image_size(self): - """Return the shape of the image (y, x).""" + """Return the size of the image (height, width).""" return [self.ops["Ly"], self.ops["Lx"]] @staticmethod From f7998d7dc492536a1f113fde279f9a5f6e2be77b Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Wed, 23 Aug 2023 15:26:55 -0700 Subject: [PATCH 05/39] added docstrings to nwbextractors --- .../extractors/nwbextractors/__init__.py | 14 ++ .../extractors/nwbextractors/nwbextractors.py | 152 +++++++++++++++--- 2 files changed, 144 insertions(+), 22 deletions(-) diff --git a/src/roiextractors/extractors/nwbextractors/__init__.py b/src/roiextractors/extractors/nwbextractors/__init__.py index e9bcaa63..0df342a9 100644 --- a/src/roiextractors/extractors/nwbextractors/__init__.py +++ b/src/roiextractors/extractors/nwbextractors/__init__.py @@ -1 +1,15 @@ +"""Imaging and segmentation extractors for NWB files. + +Modules +------- +nwbextractors + Imaging and segmentation extractors for NWB files. + +Classes +------- +NwbImagingExtractor + Extracts imaging data from NWB files. +NwbSegmentationExtractor + Extracts segmentation data from NWB files. +""" from .nwbextractors import NwbImagingExtractor, NwbSegmentationExtractor diff --git a/src/roiextractors/extractors/nwbextractors/nwbextractors.py b/src/roiextractors/extractors/nwbextractors/nwbextractors.py index 00c0d5f1..401f0602 100644 --- a/src/roiextractors/extractors/nwbextractors/nwbextractors.py +++ b/src/roiextractors/extractors/nwbextractors/nwbextractors.py @@ -1,5 +1,14 @@ +"""Imaging and segmentation extractors for NWB files. + +Classes +------- +NwbImagingExtractor + Extracts imaging data from NWB files. +NwbSegmentationExtractor + Extracts segmentation data from NWB files. +""" from pathlib import Path -from typing import Union, Optional, Iterable +from typing import Union, Optional, Iterable, Tuple import numpy as np from lazy_ops import DatasetView @@ -25,6 +34,7 @@ def temporary_deprecation_message(): + """Raise a NotImplementedError with a temporary deprecation message.""" raise NotImplementedError( "ROIExtractors no longer supports direct write to NWB. This method will be removed in a future release.\n\n" "Please install nwb-conversion-tools and import the corresponding write method from there.\n\nFor example,\n\n" @@ -34,11 +44,13 @@ def temporary_deprecation_message(): def check_nwb_install(): + """Check if pynwb is installed.""" assert HAVE_NWB, "To use the Nwb extractors, install pynwb: \n\n pip install pynwb\n\n" class NwbImagingExtractor(ImagingExtractor): - """ + """An imaging extractor for NWB files. + Class used to extract data from the NWB data format. Also implements a static method to write any format specific object to NWB. """ @@ -50,7 +62,8 @@ class NwbImagingExtractor(ImagingExtractor): installation_mesg = "To use the Nwb Extractor run:\n\n pip install pynwb\n\n" # error message when not installed def __init__(self, file_path: PathType, optical_series_name: Optional[str] = "TwoPhotonSeries"): - """ + """Create ImagingExtractor object from NWB file. + Parameters ---------- file_path: str @@ -120,21 +133,61 @@ def __init__(self, file_path: PathType, optical_series_name: Optional[str] = "Tw } def __del__(self): + """Close the NWB file.""" self.io.close() def time_to_frame(self, times: Union[FloatType, ArrayType]) -> np.ndarray: + """Convert time(s) to frame(s). + + Parameters + ---------- + times: float or array_like + Time or array of times to convert to frames. + + Returns + ------- + frames: numpy.ndarray + Array of frames corresponding to the input times. + """ if self._times is None: return ((times - self._imaging_start_time) * self.get_sampling_frequency()).astype("int64") else: return super().time_to_frame(times) def frame_to_time(self, frames: Union[IntType, ArrayType]) -> np.ndarray: + """Convert frame(s) to time(s). + + Parameters + ---------- + frames: int or array_like + Frame or array of frames to convert to times. + + Returns + ------- + times: numpy.ndarray + Array of times corresponding to the input frames. + """ if self._times is None: return (frames / self.get_sampling_frequency() + self._imaging_start_time).astype("float") else: return super().frame_to_time(frames) - def make_nwb_metadata(self, nwbfile, opts): + def make_nwb_metadata( + self, nwbfile, opts + ): # TODO: refactor to use two photon series name directly rather than via opts + """Create metadata dictionary for NWB file. + + Parameters + ---------- + nwbfile: pynwb.NWBFile + The NWBFile object associated with the metadata. + opts: object + The options object with name of TwoPhotonSeries as an attribute. + + Notes + ----- + Metadata dictionary is stored in the nwb_metadata attribute. + """ # Metadata dictionary - useful for constructing a nwb file self.nwb_metadata = dict( NWBFile=dict( @@ -151,6 +204,20 @@ def make_nwb_metadata(self, nwbfile, opts): ) def get_frames(self, frame_idxs: ArrayType, channel: Optional[int] = 0): + """Return frames from the video. + + Parameters + ---------- + frame_idxs : array-like + 2-element list of starting frame and ending frame (inclusive). + channel : int, optional + Channel index. The default is 0. + + Returns + ------- + numpy.ndarray + Array of frames. + """ # Fancy indexing is non performant for h5.py with long frame lists if frame_idxs is not None: slice_start = np.min(frame_idxs) @@ -167,6 +234,22 @@ def get_frames(self, frame_idxs: ArrayType, channel: Optional[int] = 0): return frames def get_video(self, start_frame=None, end_frame=None, channel: Optional[int] = 0) -> np.ndarray: + """Return the video frames. + + Parameters + ---------- + start_frame : int, optional + Starting frame. The default is None. + end_frame : int, optional + Ending frame. The default is None. + channel : int, optional + Channel index. The default is 0. + + Returns + ------- + numpy.ndarray + Array of frames. + """ start_frame = start_frame if start_frame is not None else 0 end_frame = end_frame if end_frame is not None else self.get_num_frames() @@ -174,49 +257,44 @@ def get_video(self, start_frame=None, end_frame=None, channel: Optional[int] = 0 video = video[start_frame:end_frame].transpose([0, 2, 1]) return video - def get_image_size(self): - return (self._num_rows, self._columns) + def get_image_size(self) -> Tuple[int, int]: + """Return the size of the video (num_rows, num_cols).""" + return (self._num_rows, self._num_cols) def get_num_frames(self): + """Return the number of frames in the video.""" return self._num_frames def get_sampling_frequency(self): + """Return the sampling frequency of the video.""" return self._sampling_frequency def get_channel_names(self): - """List of channels in the recoding. - - Returns - ------- - channel_names: list - List of strings of channel names - """ + """Return the channel names.""" return self._channel_names def get_num_channels(self): - """Total number of active channels in the recording - - Returns - ------- - no_of_channels: int - integer count of number of channels - """ + """Return the number of channels.""" return self._num_channels @staticmethod def add_devices(imaging, nwbfile, metadata): + """Add devices to the NWBFile (deprecated).""" temporary_deprecation_message() @staticmethod def add_two_photon_series(imaging, nwbfile, metadata, buffer_size=10, use_times=False): + """Add TwoPhotonSeries to NWBFile (deprecated).""" temporary_deprecation_message() @staticmethod def add_epochs(imaging, nwbfile): + """Add epochs to NWBFile (deprecated).""" temporary_deprecation_message() @staticmethod def get_nwb_metadata(imgextractor: ImagingExtractor): + """Return the metadata dictionary for the NWB file (deprecated).""" temporary_deprecation_message() @staticmethod @@ -229,10 +307,13 @@ def write_imaging( buffer_size: int = 10, use_times: bool = False, ): + """Write imaging data to NWB file (deprecated).""" temporary_deprecation_message() class NwbSegmentationExtractor(SegmentationExtractor): + """An segmentation extractor for NWB files.""" + extractor_name = "NwbSegmentationExtractor" installed = True # check at class level if installed or not is_writable = False @@ -240,8 +321,8 @@ class NwbSegmentationExtractor(SegmentationExtractor): installation_mesg = "" # error message when not installed def __init__(self, file_path: PathType): - """ - Creating NwbSegmentationExtractor object from nwb file + """Create NwbSegmentationExtractor object from nwb file. + Parameters ---------- file_path: PathType @@ -313,21 +394,32 @@ def __init__(self, file_path: PathType): self._channel_names = [i.name for i in imaging_plane.optical_channel] def __del__(self): + """Close the NWB file.""" self._io.close() def get_accepted_list(self): + """Return the list of accepted ROIs.""" if self._accepted_list is None: return list(range(self.get_num_rois())) else: return np.where(self._accepted_list == 1)[0].tolist() def get_rejected_list(self): + """Return the list of rejected ROIs.""" if self._rejected_list is not None: rej_list = np.where(self._rejected_list == 1)[0].tolist() if len(rej_list) > 0: return rej_list def get_images_dict(self): + """Return traces as a dictionary with key as the name of the ROiResponseSeries. + + Returns + ------- + images_dict: dict + dictionary with key, values representing different types of Images used in segmentation: + Mean, Correlation image + """ images_dict = super().get_images_dict() if self._segmentation_images is not None: images_dict.update( @@ -337,6 +429,19 @@ def get_images_dict(self): return images_dict def get_roi_locations(self, roi_ids: Optional[Iterable[int]] = None) -> np.ndarray: + """Returnn the locations of the Regions of Interest (ROIs). + + Parameters + ---------- + roi_ids: array_like + A list or 1D array of ids of the ROIs. Length is the number of ROIs + requested. + + Returns + ------ + roi_locs: numpy.ndarray + 2-D array: 2 X no_ROIs. The pixel ids (x,y) where the centroid of the ROI is. + """ if self._roi_locs is None: return all_ids = self.get_roi_ids() @@ -346,10 +451,12 @@ def get_roi_locations(self, roi_ids: Optional[Iterable[int]] = None) -> np.ndarr return np.array(self._roi_locs.data)[roi_idxs, tranpose_image_convention].T # h5py fancy indexing is slow def get_image_size(self): + """Return the size of the image (height, width).""" return self._image_masks.shape[:2] @staticmethod def get_nwb_metadata(sgmextractor): + """Return the metadata dictionary for the NWB file (deprecated).""" temporary_deprecation_message() @staticmethod @@ -362,4 +469,5 @@ def write_segmentation( buffer_size: int = 10, nwbfile=None, ): + """Write segmentation data to NWB file (deprecated).""" temporary_deprecation_message() From 766f242d16bd6f7acd6096e54e98aa40f759a4c1 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Wed, 23 Aug 2023 16:17:05 -0700 Subject: [PATCH 06/39] added docstrings to base segementation extractor --- src/roiextractors/segmentationextractor.py | 198 +++++++++++++-------- 1 file changed, 125 insertions(+), 73 deletions(-) diff --git a/src/roiextractors/segmentationextractor.py b/src/roiextractors/segmentationextractor.py index c0e9e212..e9767fed 100644 --- a/src/roiextractors/segmentationextractor.py +++ b/src/roiextractors/segmentationextractor.py @@ -1,3 +1,14 @@ +"""Base segmentation extractors. + +Classes +------- +SegmentationExtractor + Abstract class that contains all the meta-data and output data from the ROI segmentation operation when applied to + the pre-processed data. It also contains methods to read from and write to various data formats output from the + processing pipelines like SIMA, CaImAn, Suite2p, CNNM-E. +FrameSliceSegmentationExtractor + Class to get a lazy frame slice. +""" from abc import ABC, abstractmethod from typing import Union, Optional, Tuple, Iterable, List @@ -9,7 +20,8 @@ class SegmentationExtractor(ABC): - """ + """Abstract segmentation extractor class. + An abstract class that contains all the meta-data and output data from the ROI segmentation operation when applied to the pre-processed data. It also contains methods to read from and write to various data formats @@ -19,6 +31,7 @@ class SegmentationExtractor(ABC): """ def __init__(self): + """Create a new SegmentationExtractor from specified data type (unique to each child SegmentationExtractor).""" self._sampling_frequency = None self._times = None self._channel_names = ["OpticalChannel"] @@ -32,51 +45,45 @@ def __init__(self): @abstractmethod def get_accepted_list(self) -> list: - """ - The ids of the ROIs which are accepted after manual verification of - ROIs. + """Get a list of accepted ROI ids. Returns ------- accepted_list: list - List of accepted ROIs + List of accepted ROI ids. """ pass @abstractmethod def get_rejected_list(self) -> list: - """ - The ids of the ROIs which are rejected after manual verification of - ROIs. + """Get a list of rejected ROI ids. Returns ------- - accepted_list: list - List of rejected ROIs + rejected_list: list + List of rejected ROI ids. """ pass def get_num_frames(self) -> int: - """This function returns the number of frames in the recording. + """Get the number of frames in the recording (duration of recording). Returns ------- - num_of_frames: int - Number of frames in the recording (duration of recording). + num_frames: int + Number of frames in the recording. """ for trace in self.get_traces_dict().values(): if trace is not None and len(trace.shape) > 0: return trace.shape[0] def get_roi_locations(self, roi_ids=None) -> np.ndarray: - """ - Returns the locations of the Regions of Interest + """Get the locations of the Regions of Interest (ROIs). Parameters ---------- roi_ids: array_like - A list or 1D array of ids of the ROIs. Length is the number of ROIs - requested. + A list or 1D array of ids of the ROIs. Length is the number of ROIs requested. Returns ------ @@ -96,7 +103,8 @@ def get_roi_locations(self, roi_ids=None) -> np.ndarray: return roi_location def get_roi_ids(self) -> list: - """Returns the list of ROI ids. + """Get the list of ROI ids. + Returns ------- roi_ids: list @@ -105,13 +113,12 @@ def get_roi_ids(self) -> list: return list(range(self.get_num_rois())) def get_roi_image_masks(self, roi_ids=None) -> np.ndarray: - """Returns the image masks extracted from segmentation algorithm. + """Get the image masks extracted from segmentation algorithm. Parameters ---------- roi_ids: array_like - A list or 1D array of ids of the ROIs. Length is the number of ROIs - requested. + A list or 1D array of ids of the ROIs. Length is the number of ROIs requested. Returns ------- @@ -126,14 +133,12 @@ def get_roi_image_masks(self, roi_ids=None) -> np.ndarray: return np.stack([self._image_masks[:, :, k] for k in roi_idx_], 2) def get_roi_pixel_masks(self, roi_ids=None) -> np.array: - """ - Returns the weights applied to each of the pixels of the mask. + """Get the weights applied to each of the pixels of the mask. Parameters ---------- roi_ids: array_like - A list or 1D array of ids of the ROIs. Length is the number of ROIs - requested. + A list or 1D array of ids of the ROIs. Length is the number of ROIs requested. Returns ------- @@ -142,7 +147,6 @@ def get_roi_pixel_masks(self, roi_ids=None) -> np.array: Columns 1 and 2 are the x and y coordinates of the pixel, while the third column represents the weight of the pixel. """ - if roi_ids is None: roi_ids = range(self.get_num_rois()) @@ -150,23 +154,46 @@ def get_roi_pixel_masks(self, roi_ids=None) -> np.array: @abstractmethod def get_image_size(self) -> ArrayType: - """ - Frame size of movie ( x and y size of image). + """Get frame size of movie (height, width). Returns ------- no_rois: array_like - 2-D array: image y x image x + 2-D array: image height x image width """ pass def frame_slice(self, start_frame: Optional[int] = None, end_frame: Optional[int] = None): - """Return a new SegmentationExtractor ranging from the start_frame to the end_frame.""" + """Return a new SegmentationExtractor ranging from the start_frame to the end_frame. + + Parameters + ---------- + start_frame: int + The starting frame of the new SegmentationExtractor. + end_frame: int + The ending frame of the new SegmentationExtractor. + + Returns + ------- + frame_slice_segmentation_extractor: FrameSliceSegmentationExtractor + The frame slice segmentation extractor object. + """ return FrameSliceSegmentationExtractor(parent_segmentation=self, start_frame=start_frame, end_frame=end_frame) def get_traces(self, roi_ids=None, start_frame=None, end_frame=None, name="raw"): - """ - Return RoiResponseSeries + """Get the traces of each ROI specified by roi_ids. + + Parameters + ---------- + roi_ids: array_like + A list or 1D array of ids of the ROIs. Length is the number of ROIs requested. + start_frame: int + The starting frame of the trace. + end_frame: int + The ending frame of the trace. + name: str + The name of the trace to retrieve ex. 'raw', 'dff', 'neuropil', 'deconvolved' + Returns ------- traces: array_like @@ -183,13 +210,13 @@ def get_traces(self, roi_ids=None, start_frame=None, end_frame=None, name="raw") return np.array(traces[start_frame:end_frame, :])[:, idxs] # numpy fancy indexing is quickest def get_traces_dict(self): - """ - Returns traces as a dictionary with key as the name of the ROiResponseSeries + """Get traces as a dictionary with key as the name of the ROiResponseSeries. + Returns ------- _roi_response_dict: dict - dictionary with key, values representing different types of RoiResponseSeries - Flourescence, Neuropil, Deconvolved, Background etc + dictionary with key, values representing different types of RoiResponseSeries: + Fluorescence, Neuropil, Deconvolved, Background, etc. """ return dict( raw=self._roi_response_raw, @@ -199,33 +226,34 @@ def get_traces_dict(self): ) def get_images_dict(self): - """ - Returns traces as a dictionary with key as the name of the ROiResponseSeries + """Get images as a dictionary with key as the name of the ROiResponseSeries. + Returns ------- - _roi_response_dict: dict + _roi_image_dict: dict dictionary with key, values representing different types of Images used in segmentation: - Mean, Correlation image + Mean, Correlation image """ return dict(mean=self._image_mean, correlation=self._image_correlation) def get_image(self, name="correlation"): - """ - Return specific images: mean or correlation + """Get specific images: mean or correlation. + Parameters ---------- name:str name of the type of image to retrieve + Returns ------- - images: np.ndarray + images: numpy.ndarray """ if name not in self.get_images_dict(): raise ValueError(f"could not find {name} image, enter one of {list(self.get_images_dict().keys())}") return self.get_images_dict().get(name) def get_sampling_frequency(self): - """This function returns the sampling frequency in units of Hz. + """Get the sampling frequency in Hz. Returns ------- @@ -238,7 +266,7 @@ def get_sampling_frequency(self): return self._sampling_frequency def get_num_rois(self): - """Returns total number of Regions of Interest in the acquired images. + """Get total number of Regions of Interest (ROIs) in the acquired images. Returns ------- @@ -250,8 +278,8 @@ def get_num_rois(self): return trace.shape[1] def get_channel_names(self): - """ - Names of channels in the pipeline + """Get names of channels in the pipeline. + Returns ------- _channel_names: list @@ -260,41 +288,56 @@ def get_channel_names(self): return self._channel_names def get_num_channels(self): - """ - Number of channels in the pipeline + """Get number of channels in the pipeline. + Returns ------- num_of_channels: int + number of channels """ return len(self._channel_names) def get_num_planes(self): - """ - Returns the default number of planes of imaging for the segmentation extractor. - Defaults to 1 for all but the MultiSegmentationExtractor + """Get the default number of planes of imaging for the segmentation extractor. + + Notes + ----- + Defaults to 1 for all but the MultiSegmentationExtractor. + Returns ------- self._num_planes: int + number of planes """ return self._num_planes def set_times(self, times: ArrayType): - """Sets the recording times in seconds for each frame. + """Set the recording times in seconds for each frame. Parameters ---------- times: array-like The times in seconds for each frame + + Notes + ----- + Operates on _times attribute of the SegmentationExtractor object. """ assert len(times) == self.get_num_frames(), "'times' should have the same length of the number of frames!" self._times = np.array(times, dtype=np.float64) def has_time_vector(self) -> bool: - """Detect if the SegmentationExtractor has a time vector set or not.""" + """Detect if the SegmentationExtractor has a time vector set or not. + + Returns + ------- + has_time_vector: bool + True if the SegmentationExtractor has a time vector set, otherwise False. + """ return self._times is not None def frame_to_time(self, frames: Union[IntType, ArrayType]) -> Union[FloatType, ArrayType]: - """Returns the timing of frames in unit of seconds. + """Get the timing of frames in unit of seconds. Parameters ---------- @@ -313,8 +356,7 @@ def frame_to_time(self, frames: Union[IntType, ArrayType]) -> Union[FloatType, A @staticmethod def write_segmentation(segmentation_extractor, save_path, overwrite=False): - """ - Static method to write recording back to the native format. + """Write recording back to the native format. Parameters ---------- @@ -330,8 +372,7 @@ def write_segmentation(segmentation_extractor, save_path, overwrite=False): class FrameSliceSegmentationExtractor(SegmentationExtractor): - """ - Class to get a lazy frame slice. + """Class to get a lazy frame slice. Do not use this class directly but use `.frame_slice(...)` """ @@ -345,6 +386,17 @@ def __init__( start_frame: Optional[int] = None, end_frame: Optional[int] = None, ): + """Create a new FrameSliceSegmentationExtractor from parent SegmentationExtractor. + + Parameters + ---------- + parent_segmentation: SegmentationExtractor + The parent SegmentationExtractor object. + start_frame: int + The starting frame of the new SegmentationExtractor. + end_frame: int + The ending frame of the new SegmentationExtractor. + """ self._parent_segmentation = parent_segmentation self._start_frame = start_frame or 0 self._end_frame = end_frame or self._parent_segmentation.get_num_frames() @@ -368,10 +420,10 @@ def __init__( if getattr(self._parent_segmentation, "_times") is not None: self._times = self._parent_segmentation._times[start_frame:end_frame] - def get_accepted_list(self) -> list: + def get_accepted_list(self) -> list: # noqa: D102 return self._parent_segmentation.get_accepted_list() - def get_rejected_list(self) -> list: + def get_rejected_list(self) -> list: # noqa: D102 return self._parent_segmentation.get_rejected_list() def get_traces( @@ -380,7 +432,7 @@ def get_traces( start_frame: Optional[int] = None, end_frame: Optional[int] = None, name: str = "raw", - ) -> np.ndarray: + ) -> np.ndarray: # noqa: D102 start_frame = min(start_frame or 0, self._num_frames) end_frame = min(end_frame or self._num_frames, self._num_frames) return self._parent_segmentation.get_traces( @@ -390,7 +442,7 @@ def get_traces( name=name, ) - def get_traces_dict(self): + def get_traces_dict(self): # noqa: D102 return { trace_name: self._parent_segmentation.get_traces( start_frame=self._start_frame, end_frame=self._end_frame, name=trace_name @@ -398,32 +450,32 @@ def get_traces_dict(self): for trace_name, trace in self._parent_segmentation.get_traces_dict().items() } - def get_image_size(self) -> Tuple[int, int]: + def get_image_size(self) -> Tuple[int, int]: # noqa: D102 return tuple(self._parent_segmentation.get_image_size()) - def get_num_frames(self) -> int: + def get_num_frames(self) -> int: # noqa: D102 return self._num_frames - def get_num_rois(self): + def get_num_rois(self): # noqa: D102 return self._parent_segmentation.get_num_rois() - def get_images_dict(self) -> dict: + def get_images_dict(self) -> dict: # noqa: D102 return self._parent_segmentation.get_images_dict() - def get_image(self, name="correlation"): + def get_image(self, name="correlation"): # noqa: D102 return self._parent_segmentation.get_image(name=name) - def get_sampling_frequency(self) -> float: + def get_sampling_frequency(self) -> float: # noqa: D102 return self._parent_segmentation.get_sampling_frequency() - def get_channel_names(self) -> list: + def get_channel_names(self) -> list: # noqa: D102 return self._parent_segmentation.get_channel_names() - def get_num_channels(self) -> int: + def get_num_channels(self) -> int: # noqa: D102 return self._parent_segmentation.get_num_channels() - def get_num_planes(self): + def get_num_planes(self): # noqa: D102 return self._parent_segmentation.get_num_planes() - def get_roi_pixel_masks(self, roi_ids: Optional[ArrayLike] = None) -> List[np.ndarray]: + def get_roi_pixel_masks(self, roi_ids: Optional[ArrayLike] = None) -> List[np.ndarray]: # noqa: D102 return self._parent_segmentation.get_roi_pixel_masks(roi_ids=roi_ids) From ab5043fdd1e3e9656f7d2f52645eb0bc7a97b4f6 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 24 Aug 2023 11:25:29 -0700 Subject: [PATCH 07/39] ignore missing docstrings in pre-commit (D102) --- .pre-commit-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 38d4bd0b..9c0cae1c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,3 +14,5 @@ repos: rev: 6.3.0 hooks: - id: pydocstyle + args: + - --ignore=D102 From 020f873241fa8f247d356eebcd12dbfbd2ddf00d Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 24 Aug 2023 11:50:31 -0700 Subject: [PATCH 08/39] ignore missing docstrings in pre-commit (D1) and enforce numpydoc style --- .pre-commit-config.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9c0cae1c..4f77bafe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,4 +15,5 @@ repos: hooks: - id: pydocstyle args: - - --ignore=D102 + - --convention=numpy + - --add-ignore=D1 From e9f451b1eceb1cd059831dd42d30d03ec2f627a0 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 24 Aug 2023 11:53:19 -0700 Subject: [PATCH 09/39] remove #noqa's --- src/roiextractors/segmentationextractor.py | 30 +++++++++++----------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/roiextractors/segmentationextractor.py b/src/roiextractors/segmentationextractor.py index e9767fed..431a7426 100644 --- a/src/roiextractors/segmentationextractor.py +++ b/src/roiextractors/segmentationextractor.py @@ -86,7 +86,7 @@ def get_roi_locations(self, roi_ids=None) -> np.ndarray: A list or 1D array of ids of the ROIs. Length is the number of ROIs requested. Returns - ------ + ------- roi_locs: numpy.ndarray 2-D array: 2 X no_ROIs. The pixel ids (x,y) where the centroid of the ROI is. """ @@ -420,10 +420,10 @@ def __init__( if getattr(self._parent_segmentation, "_times") is not None: self._times = self._parent_segmentation._times[start_frame:end_frame] - def get_accepted_list(self) -> list: # noqa: D102 + def get_accepted_list(self) -> list: return self._parent_segmentation.get_accepted_list() - def get_rejected_list(self) -> list: # noqa: D102 + def get_rejected_list(self) -> list: return self._parent_segmentation.get_rejected_list() def get_traces( @@ -432,7 +432,7 @@ def get_traces( start_frame: Optional[int] = None, end_frame: Optional[int] = None, name: str = "raw", - ) -> np.ndarray: # noqa: D102 + ) -> np.ndarray: start_frame = min(start_frame or 0, self._num_frames) end_frame = min(end_frame or self._num_frames, self._num_frames) return self._parent_segmentation.get_traces( @@ -442,7 +442,7 @@ def get_traces( name=name, ) - def get_traces_dict(self): # noqa: D102 + def get_traces_dict(self): return { trace_name: self._parent_segmentation.get_traces( start_frame=self._start_frame, end_frame=self._end_frame, name=trace_name @@ -450,32 +450,32 @@ def get_traces_dict(self): # noqa: D102 for trace_name, trace in self._parent_segmentation.get_traces_dict().items() } - def get_image_size(self) -> Tuple[int, int]: # noqa: D102 + def get_image_size(self) -> Tuple[int, int]: return tuple(self._parent_segmentation.get_image_size()) - def get_num_frames(self) -> int: # noqa: D102 + def get_num_frames(self) -> int: return self._num_frames - def get_num_rois(self): # noqa: D102 + def get_num_rois(self): return self._parent_segmentation.get_num_rois() - def get_images_dict(self) -> dict: # noqa: D102 + def get_images_dict(self) -> dict: return self._parent_segmentation.get_images_dict() - def get_image(self, name="correlation"): # noqa: D102 + def get_image(self, name="correlation"): return self._parent_segmentation.get_image(name=name) - def get_sampling_frequency(self) -> float: # noqa: D102 + def get_sampling_frequency(self) -> float: return self._parent_segmentation.get_sampling_frequency() - def get_channel_names(self) -> list: # noqa: D102 + def get_channel_names(self) -> list: return self._parent_segmentation.get_channel_names() - def get_num_channels(self) -> int: # noqa: D102 + def get_num_channels(self) -> int: return self._parent_segmentation.get_num_channels() - def get_num_planes(self): # noqa: D102 + def get_num_planes(self): return self._parent_segmentation.get_num_planes() - def get_roi_pixel_masks(self, roi_ids: Optional[ArrayLike] = None) -> List[np.ndarray]: # noqa: D102 + def get_roi_pixel_masks(self, roi_ids: Optional[ArrayLike] = None) -> List[np.ndarray]: return self._parent_segmentation.get_roi_pixel_masks(roi_ids=roi_ids) From 5dddabecb37820c5dd2b325b6f5d1f7c1d76dbd8 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 24 Aug 2023 17:04:53 -0700 Subject: [PATCH 10/39] prototype script to find functions w/o docstrings (including inheritance) --- precommit_scripts/check_docstrings.py | 50 +++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 precommit_scripts/check_docstrings.py diff --git a/precommit_scripts/check_docstrings.py b/precommit_scripts/check_docstrings.py new file mode 100644 index 00000000..4475fa97 --- /dev/null +++ b/precommit_scripts/check_docstrings.py @@ -0,0 +1,50 @@ +import inspect +import os +import importlib +import roiextractors + + +def check_docstring(obj): + """Check if an object has a docstring.""" + doc = inspect.getdoc(obj) + if doc is None: + if inspect.isclass(obj) or inspect.isfunction(obj) or inspect.ismethod(obj): + print(f"{obj.__module__}.{obj.__name__} has no docstring.") + else: + print(f"{obj.__name__} has no docstring.") + + +def traverse_class(cls): + """Traverse a class and its methods.""" + for name, obj in inspect.getmembers(cls, inspect.isfunction or inspect.ismethod): + check_docstring(obj) + + +def traverse_module(module): + """Traverse all classes and functions in a module.""" + check_docstring(module) + for name, obj in inspect.getmembers(module, inspect.isclass or inspect.isfunction or inspect.ismethod): + parent_package = obj.__module__.split(".")[0] + if parent_package != "roiextractors": # avoid traversing external dependencies + continue + check_docstring(obj) + if inspect.isclass(obj): + traverse_class(obj) + + +def traverse_package(package): + """Traverse all modules and subpackages in a package.""" + for child in os.listdir(package.__path__[0]): + if child.startswith(".") or child == "__pycache__": + continue + elif child.endswith(".py"): + module_name = child[:-3] + module = importlib.import_module(f"{package.__name__}.{module_name}") + traverse_module(module) + else: # subpackage + subpackage = importlib.import_module(f"{package.__name__}.{child}") + traverse_package(subpackage) + + +if __name__ == "__main__": + traverse_package(roiextractors) From 7fabffa201c53af714687c6394bafaa29feb9b77 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Wed, 30 Aug 2023 10:17:32 -0700 Subject: [PATCH 11/39] refactored docstring check into pytest script and moved to tests --- .../test_docstrings.py | 54 +++++++++++-------- 1 file changed, 31 insertions(+), 23 deletions(-) rename precommit_scripts/check_docstrings.py => tests/test_docstrings.py (50%) diff --git a/precommit_scripts/check_docstrings.py b/tests/test_docstrings.py similarity index 50% rename from precommit_scripts/check_docstrings.py rename to tests/test_docstrings.py index 4475fa97..7e43ab8b 100644 --- a/precommit_scripts/check_docstrings.py +++ b/tests/test_docstrings.py @@ -2,49 +2,57 @@ import os import importlib import roiextractors +import pytest -def check_docstring(obj): - """Check if an object has a docstring.""" - doc = inspect.getdoc(obj) - if doc is None: - if inspect.isclass(obj) or inspect.isfunction(obj) or inspect.ismethod(obj): - print(f"{obj.__module__}.{obj.__name__} has no docstring.") - else: - print(f"{obj.__name__} has no docstring.") - - -def traverse_class(cls): - """Traverse a class and its methods.""" +def traverse_class(cls, objs): + """Traverse a class and its methods and append them to objs.""" for name, obj in inspect.getmembers(cls, inspect.isfunction or inspect.ismethod): - check_docstring(obj) + objs.append(obj) -def traverse_module(module): - """Traverse all classes and functions in a module.""" - check_docstring(module) +def traverse_module(module, objs): + """Traverse all classes and functions in a module and append them to objs.""" + objs.append(module) for name, obj in inspect.getmembers(module, inspect.isclass or inspect.isfunction or inspect.ismethod): parent_package = obj.__module__.split(".")[0] if parent_package != "roiextractors": # avoid traversing external dependencies continue - check_docstring(obj) + objs.append(obj) if inspect.isclass(obj): - traverse_class(obj) + traverse_class(obj, objs) -def traverse_package(package): - """Traverse all modules and subpackages in a package.""" +def traverse_package(package, objs): + """Traverse all modules and subpackages in a package to append all members to objs.""" for child in os.listdir(package.__path__[0]): if child.startswith(".") or child == "__pycache__": continue elif child.endswith(".py"): module_name = child[:-3] module = importlib.import_module(f"{package.__name__}.{module_name}") - traverse_module(module) + traverse_module(module, objs) else: # subpackage subpackage = importlib.import_module(f"{package.__name__}.{child}") - traverse_package(subpackage) + traverse_package(subpackage, objs) + + +objs = [] +traverse_package(roiextractors, objs) +print(objs) + + +@pytest.mark.parametrize("obj", objs) +def test_has_docstring(obj): + """Check if an object has a docstring.""" + doc = inspect.getdoc(obj) + if inspect.ismodule(obj): + msg = f"{obj.__name__} has no docstring." + else: + msg = f"{obj.__module__}.{obj.__qualname__} has no docstring." + assert doc is not None, msg if __name__ == "__main__": - traverse_package(roiextractors) + for obj in objs: + test_has_docstring(obj) From fb49473ec51561ac3e47b84fd2e9a4dd319ac21c Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Wed, 30 Aug 2023 11:58:41 -0700 Subject: [PATCH 12/39] added docstrings to base imaging extractor --- src/roiextractors/imagingextractor.py | 132 ++++++++++++++++++++------ 1 file changed, 103 insertions(+), 29 deletions(-) diff --git a/src/roiextractors/imagingextractor.py b/src/roiextractors/imagingextractor.py index 77576c25..41519e05 100644 --- a/src/roiextractors/imagingextractor.py +++ b/src/roiextractors/imagingextractor.py @@ -12,25 +12,47 @@ class ImagingExtractor(ABC): """Abstract class that contains all the meta-data and input data from the imaging data.""" def __init__(self, *args, **kwargs) -> None: + """Initialize the ImagingExtractor object.""" self._args = args self._kwargs = kwargs self._times = None @abstractmethod def get_image_size(self) -> Tuple[int, int]: + """Get the size of the video (num_rows, num_columns). + + Returns + ------- + image_size: tuple + Size of the video (num_rows, num_columns). + """ pass @abstractmethod def get_num_frames(self) -> int: + """Get the number of frames in the video. + + Returns + ------- + num_frames: int + Number of frames in the video. + """ pass @abstractmethod def get_sampling_frequency(self) -> float: + """Get the sampling frequency in Hz. + + Returns + ------- + sampling_frequency: float + Sampling frequency in Hz. + """ pass @abstractmethod def get_channel_names(self) -> list: - """List of channels in the recoding. + """Get the channel names in the recoding. Returns ------- @@ -41,25 +63,62 @@ def get_channel_names(self) -> list: @abstractmethod def get_num_channels(self) -> int: - """Total number of active channels in the recording + """Get the total number of active channels in the recording. Returns ------- - no_of_channels: int - integer count of number of channels + num_channels: int + Integer count of number of channels. """ pass def get_dtype(self) -> DtypeType: + """Get the data type of the video. + + Returns + ------- + dtype: dtype + Data type of the video. + """ return self.get_frames(frame_idxs=[0], channel=0).dtype @abstractmethod def get_video( self, start_frame: Optional[int] = None, end_frame: Optional[int] = None, channel: int = 0 ) -> np.ndarray: + """Get the video frames. + + Parameters + ---------- + start_frame: int, optional + Start frame index (inclusive). + end_frame: int, optional + End frame index (exclusive). + channel: int, optional + Channel index. + + Returns + ------- + video: numpy.ndarray + The video frames. + """ pass def get_frames(self, frame_idxs: ArrayType, channel: Optional[int] = 0) -> np.ndarray: + """Get specific video frames from indices (not necessarily continuous). + + Parameters + ---------- + frame_idxs: array-like + Indices of frames to return. + channel: int, optional + Channel index. + + Returns + ------- + frames: numpy.ndarray + The video frames. + """ assert max(frame_idxs) <= self.get_num_frames(), "'frame_idxs' exceed number of frames" if np.all(np.diff(frame_idxs) == 0): return self.get_video(start_frame=frame_idxs[0], end_frame=frame_idxs[-1]) @@ -67,17 +126,17 @@ def get_frames(self, frame_idxs: ArrayType, channel: Optional[int] = 0) -> np.nd return self.get_video(start_frame=frame_idxs[0], end_frame=frame_idxs[-1] + 1)[relative_indices, ..., channel] def frame_to_time(self, frames: Union[FloatType, np.ndarray]) -> Union[FloatType, np.ndarray]: - """This function converts user-inputted frame indexes to times with units of seconds. + """Convert user-inputted frame indices to times with units of seconds. Parameters ---------- frames: int or array-like - The frame or frames to be converted to times + The frame or frames to be converted to times. Returns ------- times: float or array-like - The corresponding times in seconds + The corresponding times in seconds. """ # Default implementation if self._times is None: @@ -86,17 +145,17 @@ def frame_to_time(self, frames: Union[FloatType, np.ndarray]) -> Union[FloatType return self._times[frames] def time_to_frame(self, times: Union[FloatType, ArrayType]) -> Union[FloatType, np.ndarray]: - """This function converts a user-inputted times (in seconds) to a frame indexes. + """Convert a user-inputted times (in seconds) to a frame indices. Parameters - ------- + ---------- times: float or array-like - The times (in seconds) to be converted to frame indexes + The times (in seconds) to be converted to frame indices. Returns ------- frames: float or array-like - The corresponding frame indexes + The corresponding frame indices. """ # Default implementation if self._times is None: @@ -105,7 +164,7 @@ def time_to_frame(self, times: Union[FloatType, ArrayType]) -> Union[FloatType, return np.searchsorted(self._times, times).astype("int64") def set_times(self, times: ArrayType) -> None: - """This function sets the recording times (in seconds) for each frame + """Set the recording times (in seconds) for each frame. Parameters ---------- @@ -116,45 +175,61 @@ def set_times(self, times: ArrayType) -> None: self._times = np.array(times).astype("float64") def has_time_vector(self) -> bool: - """Detect if the ImagingExtractor has a time vector set or not.""" + """Detect if the ImagingExtractor has a time vector set or not. + + Returns + ------- + has_times: bool + True if the ImagingExtractor has a time vector set, otherwise False. + """ return self._times is not None def copy_times(self, extractor) -> None: - """This function copies times from another extractor. + """Copy times from another extractor. Parameters ---------- extractor - The extractor from which the epochs will be copied + The extractor from which the times will be copied. """ if extractor._times is not None: self.set_times(deepcopy(extractor._times)) def frame_slice(self, start_frame: Optional[int] = None, end_frame: Optional[int] = None): - """Return a new ImagingExtractor ranging from the start_frame to the end_frame.""" + """Return a new ImagingExtractor ranging from the start_frame to the end_frame. + + Parameters + ---------- + start_frame: int, optional + Start frame index (inclusive). + end_frame: int, optional + End frame index (exclusive). + + Returns + ------- + imaging: FrameSliceImagingExtractor + The sliced ImagingExtractor object. + """ return FrameSliceImagingExtractor(parent_imaging=self, start_frame=start_frame, end_frame=end_frame) @staticmethod def write_imaging(imaging, save_path: PathType, overwrite: bool = False): - """ - Static method to write imaging. + """Write an imaging extractor to its native file structure. Parameters ---------- - imaging: ImagingExtractor object - The EXTRACT segmentation object from which an EXTRACT native format - file has to be generated. - save_path: str - path to save the native format. - overwrite: bool - If True and save_path is existing, it is overwritten + imaging : ImagingExtractor + The imaging extractor object to be saved. + save_path : str or Path + Path to save the file. + overwrite : bool, optional + If True, overwrite the file/folder if it already exists. The default is False. """ raise NotImplementedError class FrameSliceImagingExtractor(ImagingExtractor): - """ - Class to get a lazy frame slice. + """Class to get a lazy frame slice. Do not use this class directly but use `.frame_slice(...)` on an ImagingExtractor object. """ @@ -167,8 +242,7 @@ class FrameSliceImagingExtractor(ImagingExtractor): def __init__( self, parent_imaging: ImagingExtractor, start_frame: Optional[int] = None, end_frame: Optional[int] = None ): - """ - Initialize an ImagingExtractor whose frames subset the parent. + """Initialize an ImagingExtractor whose frames subset the parent. Subset is exclusive on the right bound, that is, the indexes of this ImagingExtractor range over [0, ..., end_frame-start_frame-1], which is used to resolve the index mapping in `get_frames(frame_idxs=[...])`. From 19150b253124e055043bd998afe688ca65f13238 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Wed, 30 Aug 2023 12:01:13 -0700 Subject: [PATCH 13/39] updated imagingextractor.py docstring --- src/roiextractors/imagingextractor.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/roiextractors/imagingextractor.py b/src/roiextractors/imagingextractor.py index 41519e05..66ede81a 100644 --- a/src/roiextractors/imagingextractor.py +++ b/src/roiextractors/imagingextractor.py @@ -1,4 +1,12 @@ -"""Base class definitions for all ImagingExtractors.""" +"""Base class definitions for all ImagingExtractors. + +Classes +------- +ImagingExtractor + Abstract class that contains all the meta-data and input data from the imaging data. +FrameSliceImagingExtractor + Class to get a lazy frame slice. +""" from abc import ABC, abstractmethod from typing import Union, Optional, Tuple from copy import deepcopy From c3ef801001f245c5a50b648c6ee211d35660a4c5 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 31 Aug 2023 10:37:24 -0700 Subject: [PATCH 14/39] removed redundant docstrings in suite2p --- .../suite2p/suite2psegmentationextractor.py | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/src/roiextractors/extractors/suite2p/suite2psegmentationextractor.py b/src/roiextractors/extractors/suite2p/suite2psegmentationextractor.py index f6535127..b5dfa62e 100644 --- a/src/roiextractors/extractors/suite2p/suite2psegmentationextractor.py +++ b/src/roiextractors/extractors/suite2p/suite2psegmentationextractor.py @@ -96,11 +96,9 @@ def _load_npy(self, filename, mmap_mode=None): return np.load(file_path, mmap_mode=mmap_mode, allow_pickle=mmap_mode is None) def get_accepted_list(self): - """Return a list of accepted ROI ids.""" return list(np.where(self.iscell[:, 0] == 1)[0]) def get_rejected_list(self): - """Return a list of rejected ROI ids.""" return list(np.where(self.iscell[:, 0] == 0)[0]) def _summary_image_read(self, bstr="meanImg"): @@ -134,18 +132,6 @@ def roi_locations(self): return np.array([j["med"] for j in self.stat]).T.astype(int) def get_roi_image_masks(self, roi_ids=None): - """Get image masks for all ROIs specified by roi_ids. - - Parameters - ---------- - roi_ids: list - A list of ROI ids to get image masks for. If None, all ROIs are used. - - Returns - ------- - image_masks: numpy.ndarray - A 3D numpy array of image masks with shape (y, x, len(roi_ids)). - """ if roi_ids is None: roi_idx_ = range(self.get_num_rois()) else: @@ -159,18 +145,6 @@ def get_roi_image_masks(self, roi_ids=None): ) def get_roi_pixel_masks(self, roi_ids=None): - """Get pixel masks for all ROIs specified by roi_ids. - - Parameters - ---------- - roi_ids: list - A list of ROI ids to get pixel masks for. If None, all ROIs are used. - - Returns - ------- - pixel_masks: list - A list of pixel masks for each ROI. - """ pixel_mask = [] for i in range(self.get_num_rois()): pixel_mask.append( @@ -191,7 +165,6 @@ def get_roi_pixel_masks(self, roi_ids=None): return [pixel_mask[i] for i in roi_idx_] def get_image_size(self): - """Return the size of the image (height, width).""" return [self.ops["Ly"], self.ops["Lx"]] @staticmethod From b2955f7d5f4633f0152f1e685090b0104736d628 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 31 Aug 2023 10:42:27 -0700 Subject: [PATCH 15/39] removed redundant docstrings in hdf5 --- .../hdf5imagingextractor.py | 30 ------------------- 1 file changed, 30 deletions(-) diff --git a/src/roiextractors/extractors/hdf5imagingextractor/hdf5imagingextractor.py b/src/roiextractors/extractors/hdf5imagingextractor/hdf5imagingextractor.py index b9344f0a..d6ff8e65 100644 --- a/src/roiextractors/extractors/hdf5imagingextractor/hdf5imagingextractor.py +++ b/src/roiextractors/extractors/hdf5imagingextractor/hdf5imagingextractor.py @@ -121,20 +121,6 @@ def __del__(self): self._file.close() def get_frames(self, frame_idxs: ArrayType, channel: Optional[int] = 0): - """Return frames from the video. - - Parameters - ---------- - frame_idxs : array-like - 2-element list of starting frame and ending frame (inclusive). - channel : int, optional - Channel index. The default is 0. - - Returns - ------- - numpy.ndarray - Array of frames. - """ # Fancy indexing is non performant for h5.py with long frame lists if frame_idxs is not None: slice_start = np.min(frame_idxs) @@ -150,22 +136,6 @@ def get_frames(self, frame_idxs: ArrayType, channel: Optional[int] = 0): return frames def get_video(self, start_frame=None, end_frame=None, channel: Optional[int] = 0) -> np.ndarray: - """Return the video frames. - - Parameters - ---------- - start_frame : int, optional - Starting frame. The default is None. - end_frame : int, optional - Ending frame. The default is None. - channel : int, optional - Channel index. The default is 0. - - Returns - ------- - numpy.ndarray - Array of frames. - """ return self._video.lazy_slice[start_frame:end_frame, :, :, channel].dsetread() def get_image_size(self) -> Tuple[int, int]: From 6b21e1c6d5f2af9f39f081ec01dc7a7e5de05fc1 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 31 Aug 2023 10:44:48 -0700 Subject: [PATCH 16/39] removed redundant docstrings in sima --- .../extractors/hdf5imagingextractor/hdf5imagingextractor.py | 5 ----- .../extractors/simaextractor/simasegmentationextractor.py | 3 --- 2 files changed, 8 deletions(-) diff --git a/src/roiextractors/extractors/hdf5imagingextractor/hdf5imagingextractor.py b/src/roiextractors/extractors/hdf5imagingextractor/hdf5imagingextractor.py index d6ff8e65..bd926820 100644 --- a/src/roiextractors/extractors/hdf5imagingextractor/hdf5imagingextractor.py +++ b/src/roiextractors/extractors/hdf5imagingextractor/hdf5imagingextractor.py @@ -139,23 +139,18 @@ def get_video(self, start_frame=None, end_frame=None, channel: Optional[int] = 0 return self._video.lazy_slice[start_frame:end_frame, :, :, channel].dsetread() def get_image_size(self) -> Tuple[int, int]: - """Return the size of the video (num_rows, num_cols).""" return (self._num_rows, self._num_cols) def get_num_frames(self): - """Return the number of frames in the video.""" return self._num_frames def get_sampling_frequency(self): - """Return the sampling frequency of the video.""" return self._sampling_frequency def get_channel_names(self): - """Return the channel names.""" return self._channel_names def get_num_channels(self): - """Return the number of channels.""" return self._num_channels @staticmethod diff --git a/src/roiextractors/extractors/simaextractor/simasegmentationextractor.py b/src/roiextractors/extractors/simaextractor/simasegmentationextractor.py index 638fe7fc..2060c43f 100644 --- a/src/roiextractors/extractors/simaextractor/simasegmentationextractor.py +++ b/src/roiextractors/extractors/simaextractor/simasegmentationextractor.py @@ -160,11 +160,9 @@ def _summary_image_read(self): return np.array(summary_image).T def get_accepted_list(self): - """Return the list of accepted ROIs.""" return list(range(self.get_num_rois())) def get_rejected_list(self): - """Return the list of rejected ROIs.""" return [a for a in range(self.get_num_rois()) if a not in set(self.get_accepted_list())] @staticmethod @@ -178,5 +176,4 @@ def write_segmentation(segmentation_object, savepath): raise NotImplementedError # TODO: implement write_segmentation def get_image_size(self): - """Return the size of the image (height, width).""" return self._image_masks.shape[0:2] From 3d631bc63876daa55b9b1a43ef7907e0002a49e1 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 31 Aug 2023 10:49:49 -0700 Subject: [PATCH 17/39] removed redundant docstrings in nwb --- .../extractors/nwbextractors/nwbextractors.py | 68 +------------------ 1 file changed, 3 insertions(+), 65 deletions(-) diff --git a/src/roiextractors/extractors/nwbextractors/nwbextractors.py b/src/roiextractors/extractors/nwbextractors/nwbextractors.py index 401f0602..17ad8331 100644 --- a/src/roiextractors/extractors/nwbextractors/nwbextractors.py +++ b/src/roiextractors/extractors/nwbextractors/nwbextractors.py @@ -137,36 +137,12 @@ def __del__(self): self.io.close() def time_to_frame(self, times: Union[FloatType, ArrayType]) -> np.ndarray: - """Convert time(s) to frame(s). - - Parameters - ---------- - times: float or array_like - Time or array of times to convert to frames. - - Returns - ------- - frames: numpy.ndarray - Array of frames corresponding to the input times. - """ if self._times is None: return ((times - self._imaging_start_time) * self.get_sampling_frequency()).astype("int64") else: return super().time_to_frame(times) def frame_to_time(self, frames: Union[IntType, ArrayType]) -> np.ndarray: - """Convert frame(s) to time(s). - - Parameters - ---------- - frames: int or array_like - Frame or array of frames to convert to times. - - Returns - ------- - times: numpy.ndarray - Array of times corresponding to the input frames. - """ if self._times is None: return (frames / self.get_sampling_frequency() + self._imaging_start_time).astype("float") else: @@ -204,20 +180,6 @@ def make_nwb_metadata( ) def get_frames(self, frame_idxs: ArrayType, channel: Optional[int] = 0): - """Return frames from the video. - - Parameters - ---------- - frame_idxs : array-like - 2-element list of starting frame and ending frame (inclusive). - channel : int, optional - Channel index. The default is 0. - - Returns - ------- - numpy.ndarray - Array of frames. - """ # Fancy indexing is non performant for h5.py with long frame lists if frame_idxs is not None: slice_start = np.min(frame_idxs) @@ -234,22 +196,6 @@ def get_frames(self, frame_idxs: ArrayType, channel: Optional[int] = 0): return frames def get_video(self, start_frame=None, end_frame=None, channel: Optional[int] = 0) -> np.ndarray: - """Return the video frames. - - Parameters - ---------- - start_frame : int, optional - Starting frame. The default is None. - end_frame : int, optional - Ending frame. The default is None. - channel : int, optional - Channel index. The default is 0. - - Returns - ------- - numpy.ndarray - Array of frames. - """ start_frame = start_frame if start_frame is not None else 0 end_frame = end_frame if end_frame is not None else self.get_num_frames() @@ -258,23 +204,18 @@ def get_video(self, start_frame=None, end_frame=None, channel: Optional[int] = 0 return video def get_image_size(self) -> Tuple[int, int]: - """Return the size of the video (num_rows, num_cols).""" return (self._num_rows, self._num_cols) def get_num_frames(self): - """Return the number of frames in the video.""" return self._num_frames def get_sampling_frequency(self): - """Return the sampling frequency of the video.""" return self._sampling_frequency def get_channel_names(self): - """Return the channel names.""" return self._channel_names def get_num_channels(self): - """Return the number of channels.""" return self._num_channels @staticmethod @@ -398,21 +339,19 @@ def __del__(self): self._io.close() def get_accepted_list(self): - """Return the list of accepted ROIs.""" if self._accepted_list is None: return list(range(self.get_num_rois())) else: return np.where(self._accepted_list == 1)[0].tolist() def get_rejected_list(self): - """Return the list of rejected ROIs.""" if self._rejected_list is not None: rej_list = np.where(self._rejected_list == 1)[0].tolist() if len(rej_list) > 0: return rej_list def get_images_dict(self): - """Return traces as a dictionary with key as the name of the ROiResponseSeries. + """Return traces as a dictionary with key as the name of the ROIResponseSeries. Returns ------- @@ -429,7 +368,7 @@ def get_images_dict(self): return images_dict def get_roi_locations(self, roi_ids: Optional[Iterable[int]] = None) -> np.ndarray: - """Returnn the locations of the Regions of Interest (ROIs). + """Return the locations of the Regions of Interest (ROIs). Parameters ---------- @@ -438,7 +377,7 @@ def get_roi_locations(self, roi_ids: Optional[Iterable[int]] = None) -> np.ndarr requested. Returns - ------ + ------- roi_locs: numpy.ndarray 2-D array: 2 X no_ROIs. The pixel ids (x,y) where the centroid of the ROI is. """ @@ -451,7 +390,6 @@ def get_roi_locations(self, roi_ids: Optional[Iterable[int]] = None) -> np.ndarr return np.array(self._roi_locs.data)[roi_idxs, tranpose_image_convention].T # h5py fancy indexing is slow def get_image_size(self): - """Return the size of the image (height, width).""" return self._image_masks.shape[:2] @staticmethod From 8023d521b5c5284470cd6bc19b029079f13e16ed Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 31 Aug 2023 10:51:35 -0700 Subject: [PATCH 18/39] fixed _columns vs _num_cols bug --- src/roiextractors/extractors/nwbextractors/nwbextractors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roiextractors/extractors/nwbextractors/nwbextractors.py b/src/roiextractors/extractors/nwbextractors/nwbextractors.py index 17ad8331..30c365b3 100644 --- a/src/roiextractors/extractors/nwbextractors/nwbextractors.py +++ b/src/roiextractors/extractors/nwbextractors/nwbextractors.py @@ -204,7 +204,7 @@ def get_video(self, start_frame=None, end_frame=None, channel: Optional[int] = 0 return video def get_image_size(self) -> Tuple[int, int]: - return (self._num_rows, self._num_cols) + return (self._num_rows, self._columns) # TODO: change name of _columns to _num_cols for consistency def get_num_frames(self): return self._num_frames From fa2589ed6c51b5d4a4ccc2258b125d7719311023 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 31 Aug 2023 11:27:13 -0700 Subject: [PATCH 19/39] added docstrings to multiimagingextractor --- src/roiextractors/multiimagingextractor.py | 53 ++++++++++++++++++++-- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/src/roiextractors/multiimagingextractor.py b/src/roiextractors/multiimagingextractor.py index 3d0a0f61..ddd6c6e0 100644 --- a/src/roiextractors/multiimagingextractor.py +++ b/src/roiextractors/multiimagingextractor.py @@ -1,3 +1,10 @@ +"""Defines the MultiImagingExtractor class. + +Classes +------- +MultiImagingExtractor + This class is used to combine multiple ImagingExtractor objects by frames. +""" from collections import defaultdict from typing import Tuple, List, Iterable, Optional @@ -8,16 +15,15 @@ class MultiImagingExtractor(ImagingExtractor): - """ - This class is used to combine multiple ImagingExtractor objects by frames. - """ + """Class to combine multiple ImagingExtractor objects by frames.""" extractor_name = "MultiImagingExtractor" installed = True installation_mesg = "" def __init__(self, imaging_extractors: List[ImagingExtractor]): - """ + """Initialize a MultiImagingExtractor object from a list of ImagingExtractors. + Parameters ---------- imaging_extractors: list of ImagingExtractor @@ -25,7 +31,7 @@ def __init__(self, imaging_extractors: List[ImagingExtractor]): """ super().__init__() assert isinstance(imaging_extractors, list), "Enter a list of ImagingExtractor objects as argument" - assert all(isinstance(IX, ImagingExtractor) for IX in imaging_extractors) + assert all(isinstance(imaging_extractor, ImagingExtractor) for imaging_extractor in imaging_extractors) self._imaging_extractors = imaging_extractors # Checks that properties are consistent between extractors @@ -44,6 +50,22 @@ def __init__(self, imaging_extractors: List[ImagingExtractor]): self.set_times(times=times) def _check_consistency_between_imaging_extractors(self): + """Check that essential properties are consistent between extractors so that they can be combined appropriately. + + Raises + ------ + AssertionError + If any of the properties are not consistent between extractors. + + Notes + ----- + This method checks the following properties: + - sampling frequency + - image size + - number of channels + - channel names + - data type + """ properties_to_check = dict( get_sampling_frequency="The sampling frequency", get_image_size="The size of a frame", @@ -59,6 +81,13 @@ def _check_consistency_between_imaging_extractors(self): ), f"{property_message} is not consistent over the files (found {unique_values})." def _get_times(self): + """Get all the times from the imaging extractors and combine them into a single array. + + Returns + ------- + times: numpy.ndarray + Array of times. + """ frame_indices = np.array([*range(self._start_frames[0], self._end_frames[-1])]) times = self.frame_to_time(frames=frame_indices) @@ -70,6 +99,20 @@ def _get_times(self): return times def _get_frames_from_an_imaging_extractor(self, extractor_index: int, frame_idxs: ArrayType) -> NumpyArray: + """Get frames from a single imaging extractor. + + Parameters + ---------- + extractor_index: int + Index of the imaging extractor to use. + frame_idxs: array_like + Indices of the frames to get. + + Returns + ------- + frames: numpy.ndarray + Array of frames. + """ imaging_extractor = self._imaging_extractors[extractor_index] frames = imaging_extractor.get_frames(frame_idxs=frame_idxs) return frames From 09b002695aff517d519306cbdfcb620573ece0b5 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 31 Aug 2023 11:47:19 -0700 Subject: [PATCH 20/39] added docstrings to multisegmentationextractor --- .../multisegmentationextractor.py | 62 ++++++++++++++++--- src/roiextractors/segmentationextractor.py | 2 +- 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/src/roiextractors/multisegmentationextractor.py b/src/roiextractors/multisegmentationextractor.py index 7f4bca87..9e7a3e65 100644 --- a/src/roiextractors/multisegmentationextractor.py +++ b/src/roiextractors/multisegmentationextractor.py @@ -1,9 +1,29 @@ +"""Defines the MultiSegmentationExtractor class. + +Classes +------- +MultiSegmentationExtractor + This class is used to combine multiple SegmentationExtractor objects by frames. +""" import numpy as np from .segmentationextractor import SegmentationExtractor -def concatenate_output(func): +def concatenate_output(func): # TODO: refactor to avoid magical behavior + """Concatenate output of single SegmentationExtractor methods. + + Parameters + ---------- + func: function + function to be decorated + + Returns + ------- + _get_from_roi_map: function + decorated function + """ + def _get_from_roi_map(self, roi_ids=None, **kwargs): out = [] if roi_ids is None: @@ -21,10 +41,7 @@ def _get_from_roi_map(self, roi_ids=None, **kwargs): class MultiSegmentationExtractor(SegmentationExtractor): - """ - This class is used to concatenate multi-plane recordings from the same device and session - of experiment. - """ + """Class is used to concatenate multi-plane recordings from the same device and session of experiment.""" extractor_name = "MultiSegmentationExtractor" installed = True # check at class level if installed or not @@ -32,12 +49,13 @@ class MultiSegmentationExtractor(SegmentationExtractor): mode = "file" installation_mesg = "" # error message when not installed - def __init__(self, segmentatation_extractors_list, plane_names=None): - """ + def __init__(self, segmentatation_extractors_list, plane_names=None): # TODO: Hungarian notation --> type hints + """Initialize a MultiSegmentationExtractor object from a list of SegmentationExtractors. + Parameters ---------- segmentatation_extractors_list: list of SegmentationExtractor - list of segmentation extractor objects for every plane + list of segmentation extractor objects (one for each plane) plane_names: list list of strings of names for the plane. Defaults to 'Plane0', 'Plane1' ... """ @@ -72,10 +90,24 @@ def __init__(self, segmentatation_extractors_list, plane_names=None): @property def no_planes(self): + """Number of planes in the recording. + + Returns + ------- + no_planes: int + number of planes in the recording + """ return self._no_planes @property def segmentations(self): + """List of segmentation extractors (one for each plane). + + Returns + ------- + segmentations: list + list of segmentation extractors (one for each plane) + """ return self._segmentations def get_num_channels(self): @@ -84,7 +116,19 @@ def get_num_channels(self): def get_num_rois(self): return len(self._all_roi_ids) - def get_images(self, name="correlation_plane0"): + def get_images(self, name="correlation_plane0"): # TODO: add get_images to base SegmentationExtractor class + """Get images from the imaging extractors. + + Parameters + ---------- + name: str + name of the image to get + + Returns + ------- + images: numpy.ndarray + Array of images. + """ plane_no = int(name[-1]) return self._segmentations[plane_no].get_images(name=name.split("_")[0]) diff --git a/src/roiextractors/segmentationextractor.py b/src/roiextractors/segmentationextractor.py index 431a7426..e5d85dda 100644 --- a/src/roiextractors/segmentationextractor.py +++ b/src/roiextractors/segmentationextractor.py @@ -226,7 +226,7 @@ def get_traces_dict(self): ) def get_images_dict(self): - """Get images as a dictionary with key as the name of the ROiResponseSeries. + """Get images as a dictionary with key as the name of the ROIResponseSeries. Returns ------- From d4d380c2054dc7d7b2ad2596415f645bfb3dbaef Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 31 Aug 2023 13:15:27 -0700 Subject: [PATCH 21/39] added docstrings to caiman --- .../caiman/caimansegmentationextractor.py | 61 +++++++++++++++++-- 1 file changed, 57 insertions(+), 4 deletions(-) diff --git a/src/roiextractors/extractors/caiman/caimansegmentationextractor.py b/src/roiextractors/extractors/caiman/caimansegmentationextractor.py index c13bb1ad..97ad60a3 100644 --- a/src/roiextractors/extractors/caiman/caimansegmentationextractor.py +++ b/src/roiextractors/extractors/caiman/caimansegmentationextractor.py @@ -1,3 +1,10 @@ +"""A SegmentationExtractor for CaImAn. + +Classes +------- +CaimanSegmentationExtractor + A class for extracting segmentation from CaImAn output. +""" from pathlib import Path try: @@ -22,10 +29,11 @@ class CaimanSegmentationExtractor(SegmentationExtractor): - """ + """A SegmentationExtractor for CaImAn. + This class inherits from the SegmentationExtractor class, having all its funtionality specifically applied to the dataset output from - the \'CNMF-E\' ROI segmentation method. + the 'CNMF-E' ROI segmentation method. """ extractor_name = "CaimanSegmentation" @@ -36,7 +44,8 @@ class CaimanSegmentationExtractor(SegmentationExtractor): installation_mesg = "To use the CaimanSegmentationExtractor install h5py and scipy: \n\n pip install scipy/h5py\n\n" def __init__(self, file_path: PathType): - """ + """Initialize a CaimanSegmentationExtractor instance. + Parameters ---------- file_path: str @@ -52,13 +61,28 @@ def __init__(self, file_path: PathType): self._sampling_frequency = self._dataset_file["params"]["data"]["fr"][()] self._image_masks = self._image_mask_sparse_read() - def __del__(self): + def __del__(self): # TODO: refactor segmentation extractors who use __del__ together into a base class + """Close the h5py file when the object is deleted.""" self._dataset_file.close() def _file_extractor_read(self): + """Read the h5py file. + + Returns + ------- + h5py.File + The h5py file object specified by self.file_path. + """ return h5py.File(self.file_path, "r") def _image_mask_sparse_read(self): + """Read the image masks from the h5py file. + + Returns + ------- + image_masks: numpy.ndarray + The image masks for each ROI. + """ roi_ids = self._dataset_file["estimates"]["A"]["indices"] masks = self._dataset_file["estimates"]["A"]["data"] ids = self._dataset_file["estimates"]["A"]["indptr"] @@ -70,12 +94,25 @@ def _image_mask_sparse_read(self): return image_masks def _trace_extractor_read(self, field): + """Read the traces specified by the field from the estimates dataset of the h5py file. + + Parameters + ---------- + field: str + The field to read from the estimates object. + + Returns + ------- + lazy_ops.DatasetView + The traces specified by the field. + """ lazy_ops = get_package(package_name="lazy_ops") if field in self._dataset_file["estimates"]: return lazy_ops.DatasetView(self._dataset_file["estimates"][field]).lazy_transpose() def _summary_image_read(self): + """Read the summary image (Cn) from the estimates dataset of the h5py file.""" if self._dataset_file["estimates"].get("Cn"): return np.array(self._dataset_file["estimates"]["Cn"]) @@ -97,6 +134,22 @@ def get_rejected_list(self): @staticmethod def write_segmentation(segmentation_object, save_path, overwrite=True): + """Write a segmentation object to a *.hdf5 or *.h5 file specified by save_path. + + Parameters + ---------- + segmentation_object: SegmentationExtractor + The segmentation object to be written to file. + save_path: str + The path to the file to be written. + overwrite: bool + If True, overwrite the file if it already exists. + + Raises + ------ + FileExistsError + If the file already exists and overwrite is False. + """ save_path = Path(save_path) assert save_path.suffix in [ ".hdf5", From ba33b96c083a5b162c1bda655255765a10ee3f1f Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 31 Aug 2023 13:18:47 -0700 Subject: [PATCH 22/39] added docstrings to caiman --- src/roiextractors/extractors/caiman/__init__.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/roiextractors/extractors/caiman/__init__.py b/src/roiextractors/extractors/caiman/__init__.py index 3f7fd8d9..866519c6 100644 --- a/src/roiextractors/extractors/caiman/__init__.py +++ b/src/roiextractors/extractors/caiman/__init__.py @@ -1 +1,13 @@ +"""A Segmentation Extractor for Caiman. + +Modules +------- +caimansegmentationextractor + A Segmentation Extractor for Caiman. + +Classes +------- +CaimanSegmentationExtractor + A class for extracting segmentation from Caiman output. +""" from .caimansegmentationextractor import CaimanSegmentationExtractor From 0d8e86f91909481444c0d59238c3954d211896c4 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 31 Aug 2023 13:34:43 -0700 Subject: [PATCH 23/39] added docstrings to memmap --- .../extractors/memmapextractors/__init__.py | 18 +++++++- .../memmapextractors/memmapextractors.py | 45 ++++++++++--------- .../memmapextractors/numpymemampextractor.py | 13 ++++-- 3 files changed, 52 insertions(+), 24 deletions(-) diff --git a/src/roiextractors/extractors/memmapextractors/__init__.py b/src/roiextractors/extractors/memmapextractors/__init__.py index c6d54c39..eb887aec 100644 --- a/src/roiextractors/extractors/memmapextractors/__init__.py +++ b/src/roiextractors/extractors/memmapextractors/__init__.py @@ -1,2 +1,18 @@ -from .numpymemampextractor import NumpyMemmapImagingExtractor +"""Defines memmap-based ImagingExtractors. Currently, only numpy.memmap is supported. + +Modules +------- +memmapextractors + The base class for memmapable imaging extractors. +numpymemampextractor + The class for reading optical imaging data stored in a binary format with numpy.memmap. + +Classes +------- +MemmapImagingExtractor + The base class for memmapable imaging extractors. +NumpyMemmapImagingExtractor + The class for reading optical imaging data stored in a binary format with numpy.memmap. +""" from .memmapextractors import MemmapImagingExtractor +from .numpymemampextractor import NumpyMemmapImagingExtractor diff --git a/src/roiextractors/extractors/memmapextractors/memmapextractors.py b/src/roiextractors/extractors/memmapextractors/memmapextractors.py index a92db2f0..450a5dc6 100644 --- a/src/roiextractors/extractors/memmapextractors/memmapextractors.py +++ b/src/roiextractors/extractors/memmapextractors/memmapextractors.py @@ -1,3 +1,10 @@ +"""Defines the base class for memmapable imaging extractors. + +Classes +------- +MemmapImagingExtractor + The base class for memmapable imaging extractors. +""" from pathlib import Path import numpy as np @@ -15,14 +22,20 @@ class MemmapImagingExtractor(ImagingExtractor): + """Abstract class for memmapable imaging extractors.""" + extractor_name = "MemmapImagingExtractor" def __init__( self, video, ) -> None: - """ - Abstract class for memmapable imaging extractors. + """Create a MemmapImagingExtractor instance. + + Parameters + ---------- + video: numpy.ndarray + The video data. """ self._video = video super().__init__() @@ -53,29 +66,22 @@ def get_sampling_frequency(self) -> float: return self._sampling_frequency def get_channel_names(self): - """List of channels in the recoding. - - Returns - ------- - channel_names: list - List of strings of channel names - """ pass def get_num_channels(self) -> int: - """Total number of active channels in the recording - - Returns - ------- - no_of_channels: int - integer count of number of channels - """ return self._num_channels def get_dtype(self) -> DtypeType: return self.dtype def get_video_shape(self) -> Tuple[int, int, int, int]: + """Return the shape of the video data. + + Returns + ------- + video_shape: Tuple[int, int, int, int] + The shape of the video data (num_frames, num_rows, num_columns, num_channels). + """ return (self._num_frames, self._num_rows, self._num_columns, self._num_channels) @staticmethod @@ -85,12 +91,12 @@ def write_imaging( verbose: bool = False, buffer_size_in_gb: Optional[float] = None, ) -> None: - """ - Static method to write imaging. + """Write imaging by flushing to disk. Parameters ---------- - imaging: An ImagingExtractor object that inherited from MemmapImagingExtractor + imaging_extractor: ImagingExtractor + An ImagingExtractor object that inherited from MemmapImagingExtractor save_path: str path to save the native format to. verbose: bool @@ -98,7 +104,6 @@ def write_imaging( buffer_size_in_gb: float The size of the buffer in Gigabytes. The default of None results in buffering over one frame at a time. """ - # The base and default case is to load one image at a time. if buffer_size_in_gb is None: buffer_size_in_gb = 0 diff --git a/src/roiextractors/extractors/memmapextractors/numpymemampextractor.py b/src/roiextractors/extractors/memmapextractors/numpymemampextractor.py index a453cb47..a2510c80 100644 --- a/src/roiextractors/extractors/memmapextractors/numpymemampextractor.py +++ b/src/roiextractors/extractors/memmapextractors/numpymemampextractor.py @@ -1,3 +1,10 @@ +"""NumpyMemmapImagingExtractor class. + +Classes +------- +NumpyMemmapImagingExtractor + The class for reading optical imaging data stored in a binary format with numpy.memmap. +""" import os from pathlib import Path from typing import Tuple, Dict @@ -12,6 +19,8 @@ class NumpyMemmapImagingExtractor(MemmapImagingExtractor): + """An ImagingExtractor class for reading optical imaging data stored in a binary format with numpy.memmap.""" + extractor_name = "NumpyMemmapImagingExtractor" def __init__( @@ -22,8 +31,7 @@ def __init__( dtype: DtypeType, offset: int = 0, ): - """Class for reading optical imaging data stored in a binary format with np.memmap - + """Create an instance of NumpyMemmapImagingExtractor. Parameters ---------- @@ -63,7 +71,6 @@ def __init__( offset : int, optional The offset in bytes. Usually corresponds to the number of bytes occupied by the header. 0 by default. """ - self.installed = True self.file_path = Path(file_path) From 9299281d938791a6b00060a2a893c08e14510b8c Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 31 Aug 2023 14:24:09 -0700 Subject: [PATCH 24/39] added docstrings to miniscope --- .../miniscopeimagingextractor/__init__.py | 12 +++++ .../miniscopeimagingextractor.py | 52 ++++++++++++++++--- 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/src/roiextractors/extractors/miniscopeimagingextractor/__init__.py b/src/roiextractors/extractors/miniscopeimagingextractor/__init__.py index a9b783a6..43661b3a 100644 --- a/src/roiextractors/extractors/miniscopeimagingextractor/__init__.py +++ b/src/roiextractors/extractors/miniscopeimagingextractor/__init__.py @@ -1 +1,13 @@ +"""An ImagingExtractor for the Miniscope video (.avi) format. + +Modules +------- +miniscopeimagingextractor + An ImagingExtractor for the Miniscope video (.avi) format. + +Classes +------- +MiniscopeImagingExtractor + An ImagingExtractor for the Miniscope video (.avi) format. +""" from .miniscopeimagingextractor import MiniscopeImagingExtractor diff --git a/src/roiextractors/extractors/miniscopeimagingextractor/miniscopeimagingextractor.py b/src/roiextractors/extractors/miniscopeimagingextractor/miniscopeimagingextractor.py index 12bcfacd..d7d293ac 100644 --- a/src/roiextractors/extractors/miniscopeimagingextractor/miniscopeimagingextractor.py +++ b/src/roiextractors/extractors/miniscopeimagingextractor/miniscopeimagingextractor.py @@ -1,3 +1,10 @@ +"""MiniscopeImagingExtractor class. + +Classes +------- +MiniscopeImagingExtractor + An ImagingExtractor for the Miniscope video (.avi) format. +""" import json import re from pathlib import Path @@ -10,16 +17,19 @@ from ...extraction_tools import PathType, DtypeType, get_package -class MiniscopeImagingExtractor(MultiImagingExtractor): +class MiniscopeImagingExtractor(MultiImagingExtractor): # TODO: rename to MiniscopeMultiImagingExtractor + """An ImagingExtractor for the Miniscope video (.avi) format. + + This format consists of video (.avi) file(s) and configuration files (.json). + One _MiniscopeImagingExtractor is created for each video file and then combined into the MiniscopeImagingExtractor. + """ + extractor_name = "MiniscopeImaging" is_writable = True mode = "folder" def __init__(self, folder_path: PathType): - """ - The imaging extractor for the Miniscope video (.avi) format. - This format consists of video (.avi) file(s) and configuration files (.json). - + """Create a MiniscopeImagingExtractor instance from a folder path. Parameters ---------- @@ -55,9 +65,22 @@ def __init__(self, folder_path: PathType): class _MiniscopeImagingExtractor(ImagingExtractor): + """An ImagingExtractor for the Miniscope video (.avi) format. + + This format consists of a single video (.avi) file and configuration file (.json). + Multiple _MiniscopeImagingExtractor are combined into the MiniscopeImagingExtractor for public access. + """ + extractor_name = "_MiniscopeImaging" def __init__(self, file_path: PathType): + """Create a _MiniscopeImagingExtractor instance from a file path. + + Parameters + ---------- + file_path: PathType + The file path to the Miniscope video (.avi) file. + """ from neuroconv.datainterfaces.behavior.video.video_utils import VideoCaptureContext self._video_capture = VideoCaptureContext @@ -93,7 +116,24 @@ def get_channel_names(self) -> List[str]: def get_video( self, start_frame: Optional[int] = None, end_frame: Optional[int] = None, channel: int = 0 ) -> np.ndarray: - """ + """Get the video frames. + + Parameters + ---------- + start_frame: int, optional + Start frame index (inclusive). + end_frame: int, optional + End frame index (exclusive). + channel: int, optional + Channel index. + + Returns + ------- + video: numpy.ndarray + The video frames. + + Notes + ----- The grayscale conversion is based on minian https://github.com/denisecailab/minian/blob/f64c456ca027200e19cf40a80f0596106918fd09/minian/utilities.py#LL272C12-L272C12 """ From 3d928a6f04d97c08d0d945be0eef7fb405efeb33 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 31 Aug 2023 16:07:08 -0700 Subject: [PATCH 25/39] added docstrings to numpy --- .../extractors/numpyextractors/__init__.py | 14 ++++ .../numpyextractors/numpyextractors.py | 78 ++++++++++++++----- 2 files changed, 74 insertions(+), 18 deletions(-) diff --git a/src/roiextractors/extractors/numpyextractors/__init__.py b/src/roiextractors/extractors/numpyextractors/__init__.py index 584143a0..be6df5a4 100644 --- a/src/roiextractors/extractors/numpyextractors/__init__.py +++ b/src/roiextractors/extractors/numpyextractors/__init__.py @@ -1 +1,15 @@ +"""Imaging and Segmenation Extractors for .npy files. + +Modules +------- +numpyextractors + Imaging and Segmenation Extractors for .npy files. + +Classes +------- +NumpyImagingExtractor + An ImagingExtractor specified by timeseries .npy file, sampling frequency, and channel names. +NumpySegmentationExtractor + A Segmentation extractor specified by image masks and traces .npy files. +""" from .numpyextractors import NumpyImagingExtractor, NumpySegmentationExtractor diff --git a/src/roiextractors/extractors/numpyextractors/numpyextractors.py b/src/roiextractors/extractors/numpyextractors/numpyextractors.py index 2a184de4..42862268 100644 --- a/src/roiextractors/extractors/numpyextractors/numpyextractors.py +++ b/src/roiextractors/extractors/numpyextractors/numpyextractors.py @@ -1,3 +1,12 @@ +"""Imaging and Segmenation Extractors for .npy files. + +Classes +------- +NumpyImagingExtractor + An ImagingExtractor specified by timeseries .npy file, sampling frequency, and channel names. +NumpySegmentationExtractor + A Segmentation extractor specified by image masks and traces .npy files. +""" from pathlib import Path from typing import Optional, Tuple @@ -10,6 +19,8 @@ class NumpyImagingExtractor(ImagingExtractor): + """An ImagingExtractor specified by timeseries .npy file, sampling frequency, and channel names.""" + extractor_name = "NumpyImagingExtractor" installed = True is_writable = True @@ -21,6 +32,17 @@ def __init__( sampling_frequency: FloatType, channel_names: ArrayType = None, ): + """Create a NumpyImagingExtractor from a .npy file. + + Parameters + ---------- + timeseries: PathType + Path to .npy file. + sampling_frequency: FloatType + Sampling frequency of the video in Hz. + channel_names: ArrayType + List of channel names. + """ ImagingExtractor.__init__(self) if isinstance(timeseries, (str, Path)): @@ -34,7 +56,7 @@ def __init__( "sampling_frequency": sampling_frequency, } else: - raise ValueError("'timeeseries' is does not exist") + raise ValueError("'timeseries' is does not exist") elif isinstance(timeseries, np.ndarray): self.is_dumpable = False self._video = timeseries @@ -91,27 +113,24 @@ def get_sampling_frequency(self): return self._sampling_frequency def get_channel_names(self): - """List of channels in the recoding. - - Returns - ------- - channel_names: list - List of strings of channel names - """ return self._channel_names def get_num_channels(self): - """Total number of active channels in the recording - - Returns - ------- - no_of_channels: int - integer count of number of channels - """ return self._num_channels @staticmethod def write_imaging(imaging, save_path, overwrite: bool = False): + """Write a NumpyImagingExtractor to a .npy file. + + Parameters + ---------- + imaging: NumpyImagingExtractor + The imaging extractor object to be written to file. + save_path: str or PathType + Path to .npy file. + overwrite: bool + If True, overwrite file if it already exists. + """ save_path = Path(save_path) assert save_path.suffix == ".npy", "'save_path' should have a .npy extension" @@ -125,7 +144,8 @@ def write_imaging(imaging, save_path, overwrite: bool = False): class NumpySegmentationExtractor(SegmentationExtractor): - """ + """A Segmentation extractor specified by image masks and traces .npy files. + NumpySegmentationExtractor objects are built to contain all data coming from a file format for which there is currently no support. To construct this, all data must be entered manually as arguments. @@ -154,8 +174,9 @@ def __init__( channel_names=None, movie_dims=None, ): - """ - Parameters: + """Create a NumpySegmentationExtractor from a .npy file. + + Parameters ---------- image_masks: np.ndarray Binary image for each of the regions of interest @@ -283,6 +304,13 @@ def __init__( @property def image_dims(self): + """Return the dimensions of the image. + + Returns + ------- + image_dims: list + The dimensions of the image (num_rois, num_rows, num_columns). + """ return list(self._image_masks.shape[0:2]) def get_accepted_list(self): @@ -299,6 +327,7 @@ def get_rejected_list(self): @property def roi_locations(self): + """Returns the center locations (x, y) of each ROI.""" if self._roi_locs is None: num_ROIs = self.get_num_rois() raw_images = self._image_masks @@ -312,6 +341,19 @@ def roi_locations(self): @staticmethod def write_segmentation(segmentation_object, save_path): + """Write a NumpySegmentationExtractor to a .npy file. + + Parameters + ---------- + segmentation_object: NumpySegmentationExtractor + The segmentation extractor object to be written to file. + save_path: str or PathType + Path to .npy file. + + Notes + ----- + This method is not implemented yet. + """ raise NotImplementedError # defining the abstract class informed methods: From e574d8f98fb28d9383b184036bb1438cf9de447d Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 31 Aug 2023 16:14:20 -0700 Subject: [PATCH 26/39] added docstrings to scanbox image --- .../sbximagingextractor/__init__.py | 12 +++++ .../sbximagingextractor.py | 50 ++++++++++++++++++- 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/src/roiextractors/extractors/sbximagingextractor/__init__.py b/src/roiextractors/extractors/sbximagingextractor/__init__.py index 1f76a590..0671dd41 100644 --- a/src/roiextractors/extractors/sbximagingextractor/__init__.py +++ b/src/roiextractors/extractors/sbximagingextractor/__init__.py @@ -1 +1,13 @@ +"""A segmentation extractor for Scanbox imaging data. + +Modules +------- +sbximagingextractor + A segmentation extractor for Scanbox imaging data. + +Classes +------- +SbxImagingExtractor + An ImagingExtractor for Scanbox Image files. +""" from .sbximagingextractor import SbxImagingExtractor diff --git a/src/roiextractors/extractors/sbximagingextractor/sbximagingextractor.py b/src/roiextractors/extractors/sbximagingextractor/sbximagingextractor.py index e8cb2415..bfca09ed 100644 --- a/src/roiextractors/extractors/sbximagingextractor/sbximagingextractor.py +++ b/src/roiextractors/extractors/sbximagingextractor/sbximagingextractor.py @@ -1,3 +1,10 @@ +"""Imaging Extractors for Scanbox files. + +Classes +------- +SbxImagingExtractor + An ImagingExtractor for Scanbox Image files. +""" from multiprocessing.sharedctypes import Value import os from pathlib import Path @@ -18,6 +25,8 @@ class SbxImagingExtractor(ImagingExtractor): + """Imaging extractor for the Scanbox image format.""" + extractor_name = "SbxImaging" installed = HAVE_Scipy # check at class level if installed or not is_writable = True @@ -25,7 +34,7 @@ class SbxImagingExtractor(ImagingExtractor): installation_mesg = "To use the Sbx Extractor run:\n\n pip install scipy\n\n" # error message when not installed def __init__(self, file_path: PathType, sampling_frequency: Optional[float] = None): - """Imaging extractor for the Scanbox image format + """Create a SbxImagingExtractor from .mat or .sbx files. Parameters ---------- @@ -59,6 +68,20 @@ def __init__(self, file_path: PathType, sampling_frequency: Optional[float] = No @staticmethod def _return_mat_and_sbx_filepaths(file_path): + """Return the `.mat` and `.sbx` file paths from a given file path pointing to either of them. + + Parameters + ---------- + file_path : str or python Path objects + The file path pointing to a file in either `.mat` or `.sbx` format. + + Returns + ------- + mat_file_path : str or python Path object + The file path pointing to the `.mat` file. + sbx_file_path : str or python Path object + The file path pointing to the `.sbx` file. + """ file_path = Path(file_path) if file_path.suffix not in [".mat", ".sbx"]: assertion_msg = "File path not pointing to a `.sbx` or `.mat` file" @@ -69,7 +92,8 @@ def _return_mat_and_sbx_filepaths(file_path): return mat_file_path, sbx_file_path def _loadmat(self): - """ + """Load matlab .mat file. + this function should be called instead of direct spio.loadmat as it cures the problem of not properly recovering python dictionaries from mat files. It calls the function check keys to cure all entries @@ -127,6 +151,13 @@ def _loadmat(self): return info def _sbx_read(self): + """Read the `.sbx` file and return a numpy array. + + Returns + ------- + np_data : np.ndarray + The numpy array containing the data from the `.sbx` file. + """ nrows = self._info["recordsPerBuffer"] ncols = self._info["sz"][1] nchannels = self._info["nChan"] @@ -166,4 +197,19 @@ def get_num_channels(self) -> int: @staticmethod def write_imaging(imaging, save_path: PathType, overwrite: bool = False): + """Write a SbxImagingExtractor to a `.mat` file. + + Parameters + ---------- + imaging : SbxImagingExtractor + The imaging extractor object to be written to a `.mat` file. + save_path : str or python Path object + The path to the `.mat` file to be written. + overwrite : bool, optional + If True, the `.mat` file will be overwritten if it already exists. + + Notes + ----- + This function is not implemented yet. + """ raise NotImplementedError From 30f71fbc0e3cf9182dcd636863f53e72b92a43d3 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 31 Aug 2023 16:21:48 -0700 Subject: [PATCH 27/39] added docstring to multisegmentationextractor decorator --- src/roiextractors/multisegmentationextractor.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/roiextractors/multisegmentationextractor.py b/src/roiextractors/multisegmentationextractor.py index 9e7a3e65..57706c5a 100644 --- a/src/roiextractors/multisegmentationextractor.py +++ b/src/roiextractors/multisegmentationextractor.py @@ -25,6 +25,20 @@ def concatenate_output(func): # TODO: refactor to avoid magical behavior """ def _get_from_roi_map(self, roi_ids=None, **kwargs): + """Call member function of each SegmentationExtractor specified by func and concatenate the output. + + Parameters + ---------- + roi_ids: list + list of roi ids to be used + kwargs: dict + keyword arguments to be passed to func + + Returns + ------- + out: list + list of outputs from each SegmentationExtractor + """ out = [] if roi_ids is None: roi_ids = np.array(self._all_roi_ids) From cf6ef6205d4bb36ba593218d4d0d38c6504f753d Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 31 Aug 2023 17:36:44 -0700 Subject: [PATCH 28/39] added docstrings to EXTRACT and CNMF-E --- .../caiman/caimansegmentationextractor.py | 2 +- .../extractors/schnitzerextractor/__init__.py | 22 ++- .../cnmfesegmentationextractor.py | 78 ++++++++- .../extractsegmentationextractor.py | 153 +++++++++++------- 4 files changed, 194 insertions(+), 61 deletions(-) diff --git a/src/roiextractors/extractors/caiman/caimansegmentationextractor.py b/src/roiextractors/extractors/caiman/caimansegmentationextractor.py index 97ad60a3..1c1c5b64 100644 --- a/src/roiextractors/extractors/caiman/caimansegmentationextractor.py +++ b/src/roiextractors/extractors/caiman/caimansegmentationextractor.py @@ -33,7 +33,7 @@ class CaimanSegmentationExtractor(SegmentationExtractor): This class inherits from the SegmentationExtractor class, having all its funtionality specifically applied to the dataset output from - the 'CNMF-E' ROI segmentation method. + the 'CaImAn' ROI segmentation method. """ extractor_name = "CaimanSegmentation" diff --git a/src/roiextractors/extractors/schnitzerextractor/__init__.py b/src/roiextractors/extractors/schnitzerextractor/__init__.py index 7a10c775..e73a77f3 100644 --- a/src/roiextractors/extractors/schnitzerextractor/__init__.py +++ b/src/roiextractors/extractors/schnitzerextractor/__init__.py @@ -1,6 +1,26 @@ +"""Segmentation extractors for CNMF-E and EXTRACT ROI segmentation method. + +Modules +------- +cnmfesegmentationextractor + A segmentation extractor for CNMF-E ROI segmentation method. +extractsegmentationextractor + A segmentation extractor for EXTRACT segmentation method. + +Classes +------- +CnmfeSegmentationExtractor + A segmentation extractor for CNMF-E ROI segmentation method. +ExtractSegmentationExtractor + Abstract class that defines which EXTRACT class to use for a given file (new vs old). +NewExtractSegmentationExtractor + Extractor for reading the segmentation data that results from calls to newer versions of EXTRACT. +LegacyExtractSegmentationExtractor + Extractor for reading the segmentation data that results from calls to older versions of EXTRACT. +""" from .cnmfesegmentationextractor import CnmfeSegmentationExtractor from .extractsegmentationextractor import ( LegacyExtractSegmentationExtractor, ExtractSegmentationExtractor, NewExtractSegmentationExtractor, -) +) # TODO: remove legacy imports diff --git a/src/roiextractors/extractors/schnitzerextractor/cnmfesegmentationextractor.py b/src/roiextractors/extractors/schnitzerextractor/cnmfesegmentationextractor.py index 0a9ca653..300cb574 100644 --- a/src/roiextractors/extractors/schnitzerextractor/cnmfesegmentationextractor.py +++ b/src/roiextractors/extractors/schnitzerextractor/cnmfesegmentationextractor.py @@ -1,3 +1,10 @@ +"""A segmentation extractor for CNMF-E ROI segmentation method. + +Classes +------- +CnmfeSegmentationExtractor + A segmentation extractor for CNMF-E ROI segmentation method. +""" from pathlib import Path try: @@ -23,10 +30,11 @@ class CnmfeSegmentationExtractor(SegmentationExtractor): - """ + """A segmentation extractor for CNMF-E ROI segmentation method. + This class inherits from the SegmentationExtractor class, having all its funtionality specifically applied to the dataset output from - the \'CNMF-E\' ROI segmentation method. + the 'CNMF-E' ROI segmentation method. """ extractor_name = "CnmfeSegmentation" @@ -36,7 +44,8 @@ class CnmfeSegmentationExtractor(SegmentationExtractor): installation_mesg = "To use Cnmfe install h5py: \n\n pip install h5py \n\n" # error message when not installed def __init__(self, file_path: PathType): - """ + """Create a CnmfeSegmentationExtractor from a .mat file. + Parameters ---------- file_path: str @@ -53,28 +62,73 @@ def __init__(self, file_path: PathType): self._image_correlation = self._summary_image_read() def __del__(self): + """Close the file when the object is deleted.""" self._dataset_file.close() def _file_extractor_read(self): + """Read the .mat file and return the file object and the group. + + Returns + ------- + f: h5py.File + The file object. + _group0: list + Group of relevant segmentation objects. + """ f = h5py.File(self.file_path, "r") _group0_temp = list(f.keys()) _group0 = [a for a in _group0_temp if "#" not in a] return f, _group0 def _image_mask_extractor_read(self): + """Read the image masks from the .mat file and return the image masks. + + Returns + ------- + DatasetView + The image masks. + """ return DatasetView(self._dataset_file[self._group0[0]]["extractedImages"]).lazy_transpose([1, 2, 0]) def _trace_extractor_read(self): + """Read the traces from the .mat file and return the traces. + + Returns + ------- + DatasetView + The traces. + """ return self._dataset_file[self._group0[0]]["extractedSignals"] def _tot_exptime_extractor_read(self): + """Read the total experiment time from the .mat file and return the total experiment time. + + Returns + ------- + tot_exptime: float + The total experiment time. + """ return self._dataset_file[self._group0[0]]["time"]["totalTime"][0][0] def _summary_image_read(self): + """Read the summary image from the .mat file and return the summary image (Cn). + + Returns + ------- + summary_image: np.ndarray + The summary image (Cn). + """ summary_image = self._dataset_file[self._group0[0]]["Cn"] return np.array(summary_image) def _raw_datafile_read(self): + """Read the raw data file location from the .mat file and return the raw data file location. + + Returns + ------- + raw_datafile: str + The raw data file location. + """ if self._dataset_file[self._group0[0]].get("movieList"): charlist = [chr(i) for i in np.squeeze(self._dataset_file[self._group0[0]]["movieList"][:])] return "".join(charlist) @@ -88,6 +142,24 @@ def get_rejected_list(self): @staticmethod def write_segmentation(segmentation_object: SegmentationExtractor, save_path, overwrite=True): + """Write a segmentation object to a .mat file. + + Parameters + ---------- + segmentation_object: SegmentationExtractor + The segmentation object to be written. + save_path: str + The location of the folder to save the dataset.mat file. + overwrite: bool + If True, overwrite the file if it already exists. + + Raises + ------ + FileExistsError + If the file already exists and overwrite is False. + AssertionError + If save_path is not a *.mat file. + """ assert HAVE_SCIPY and HAVE_H5PY, "To use Cnmfe install scipy/h5py: \n\n pip install scipy/h5py \n\n" save_path = Path(save_path) assert save_path.suffix == ".mat", "'save_path' must be a *.mat file" diff --git a/src/roiextractors/extractors/schnitzerextractor/extractsegmentationextractor.py b/src/roiextractors/extractors/schnitzerextractor/extractsegmentationextractor.py index 0dcea87a..689966b3 100644 --- a/src/roiextractors/extractors/schnitzerextractor/extractsegmentationextractor.py +++ b/src/roiextractors/extractors/schnitzerextractor/extractsegmentationextractor.py @@ -1,4 +1,14 @@ -"""Extractor for reading the segmentation data that results from calls to EXTRACT.""" +"""Extractor for reading the segmentation data that results from calls to EXTRACT. + +Classes +------- +ExtractSegmentationExtractor + Abstract class that defines which extractor class to use for a given file. +NewExtractSegmentationExtractor + Extractor for reading the segmentation data that results from calls to newer versions of EXTRACT. +LegacyExtractSegmentationExtractor + Extractor for reading the segmentation data that results from calls to older versions of EXTRACT. +""" from abc import ABC from pathlib import Path from typing import Optional @@ -38,6 +48,7 @@ def __new__( output_struct_name: Optional[str] = None, ): """Abstract class that defines which extractor class to use for a given file. + For newer versions of the EXTRACT algorithm, the extractor class redirects to NewExtractSegmentationExtractor. For older versions, the extractor class redirects to LegacyExtractSegmentationExtractor. @@ -90,9 +101,18 @@ def _assert_file_is_mat(self): def _get_default_output_struct_name_from_file(self): """Return the default value for 'output_struct_name' when it is unspecified. + + Returns + ------- + output_struct_name: str + The name of output struct in the .mat file. + + Notes + ----- For newer version of extract, the default name is assumed to be "output". For older versions the default is "extractAnalysisOutput". - If none of them is found, raise an error that 'output_struct_name' must be supplied.""" + If none of them is found, raise an error that 'output_struct_name' must be supplied. + """ newer_default_output_struct_name = "output" legacy_default_output_struct_name = "extractAnalysisOutput" with h5py.File(name=self.file_path, mode="r") as mat_file: @@ -104,7 +124,7 @@ def _get_default_output_struct_name_from_file(self): raise AssertionError("The 'output_struct_name' must be supplied.") def _assert_output_struct_name_is_in_file(self): - """Check that 'output_struct_name' is in the file, raises an error if not.""" + """Check that 'output_struct_name' is in the file, raise an error if not.""" with h5py.File(name=self.file_path, mode="r") as mat_file: assert ( self.output_struct_name in mat_file @@ -112,8 +132,12 @@ def _assert_output_struct_name_is_in_file(self): def _check_extract_file_version(self) -> bool: """Check the version of the extract file. - If the file was created with a newer version of the EXTRACT algorithm, the - function will return True, otherwise it will return False.""" + + Returns + ------- + True if the file was created with a newer version of the EXTRACT algorithm, + False otherwise. + """ with h5py.File(name=self.file_path, mode="r") as mat_file: dataset_version = mat_file[self.output_struct_name]["info"]["version"][:] dataset_version = np.ravel(dataset_version) @@ -123,11 +147,14 @@ def _check_extract_file_version(self) -> bool: return version.Version(version_name) >= version.Version("1.0.0") -class NewExtractSegmentationExtractor(SegmentationExtractor): - """ +class NewExtractSegmentationExtractor( + SegmentationExtractor +): # TODO: refactor to inherit from LegacyExtractSegmentationExtractor + """Extractor for reading the segmentation data that results from calls to newer versions of EXTRACT. + This class inherits from the SegmentationExtractor class, having all its functionality specifically applied to the dataset output from - the \'EXTRACT\' ROI segmentation method. + the 'EXTRACT' ROI segmentation method. """ extractor_name = "NewExtractSegmentation" @@ -145,9 +172,7 @@ def __init__( sampling_frequency: float, output_struct_name: str = "output", ): - """ - Load a SegmentationExtractor from a .mat file containing the output and config structs of the EXTRACT algorithm. - For regular timing, supply the sampling frequency. For irregular timing, supply the timestamps. + """Load a SegmentationExtractor from a .mat file containing the output and config structs of the EXTRACT algorithm. Parameters ---------- @@ -158,6 +183,10 @@ def __init__( output_struct_name: str, optional The user has control over the names of the variables that return from `extraction(images, config)`. The tutorials for EXTRACT follow the naming convention of 'output', which we assume as the default. + + Notes + ----- + For regular timing, supply the sampling frequency. For irregular timing, supply the timestamps. """ super().__init__() @@ -191,9 +220,11 @@ def __init__( self.config.update(version=_decode_h5py_array(extract_version)) def close(self): + """Close the file when the object is deleted.""" self._dataset_file.close() def _file_extractor_read(self): + """Read the .mat file and return the file object.""" return h5py.File(self.file_path, "r") def _config_struct_to_dict(self, config_struct: h5py.Group) -> dict: @@ -210,70 +241,41 @@ def _config_struct_to_dict(self, config_struct: h5py.Group) -> dict: return config_dict def _image_mask_extractor_read(self) -> DatasetView: - """Returns the image masks with a shape of height, width, number of ROIs.""" + """Read the image masks from the .mat file and return the image masks. + + Returns + ------- + image_masks : DatasetView + 3-D array: height x width x number of ROIs + """ return DatasetView(self._output_struct["spatial_weights"]).lazy_transpose() def _trace_extractor_read(self) -> DatasetView: - """Returns the traces with a shape of number of frames and number of ROIs.""" - return DatasetView(self._output_struct["temporal_weights"]).lazy_transpose() - - def get_accepted_list(self) -> list: - """ - The ids of the ROIs which are accepted after manual verification of - ROIs. + """Read the traces from the .mat file and return the traces. Returns ------- - accepted_list: list - List of accepted ROIs + traces : DatasetView + 2-D array: number of frames x number of ROIs """ + return DatasetView(self._output_struct["temporal_weights"]).lazy_transpose() + + def get_accepted_list(self) -> list: return [roi for roi in self.get_roi_ids() if np.any(self._image_masks[..., roi])] def get_rejected_list(self) -> list: - """ - The ids of the ROIs which are rejected after manual verification of - ROIs. - - Returns - ------- - rejected_list: list - List of rejected ROIs - """ accepted_list = self.get_accepted_list() rejected_list = list(set(self.get_roi_ids()) - set(accepted_list)) return rejected_list def get_roi_ids(self) -> list: - """Returns the list of ROI ids. - - Returns - ------- - roi_ids: list - ROI ids list. - """ return list(range(self.get_num_rois())) def get_image_size(self) -> ArrayType: - """ - Frame size of movie (height and width of image). - - Returns - ------- - image_size: array_like - 2-D array: image height x image width - """ return self._image_masks.shape[:-1] def get_images_dict(self): - """ - Returns a dictionary with key, values representing different types of Images - used in segmentation. The shape of images is height and width. - Returns - ------- - images_dict: dict - dictionary with key, values representing different types of Images - """ images_dict = super().get_images_dict() images_dict.update( summary_image=self._info_struct["summary_image"][:].T, @@ -285,10 +287,11 @@ def get_images_dict(self): class LegacyExtractSegmentationExtractor(SegmentationExtractor): - """ + """Extractor for reading the segmentation data that results from calls to older versions of EXTRACT. + This class inherits from the SegmentationExtractor class, having all its funtionality specifically applied to the dataset output from - the \'EXTRACT\' ROI segmentation method. + the 'EXTRACT' ROI segmentation method. """ extractor_name = "LegacyExtractSegmentation" @@ -302,7 +305,8 @@ def __init__( file_path: PathType, output_struct_name: str = "extractAnalysisOutput", ): - """ + """Create a LegacyExtractSegmentationExtractor from a .mat file. + Parameters ---------- file_path: str @@ -322,25 +326,62 @@ def __init__( self._image_correlation = self._summary_image_read() def __del__(self): + """Close the file when the object is deleted.""" self._dataset_file.close() def _file_extractor_read(self): + """Read the .mat file and return the file object.""" return h5py.File(self.file_path, "r") def _image_mask_extractor_read(self): + """Read the image masks from the .mat file and return the image masks. + + Returns + ------- + image_masks : DatasetView + 3-D array: height x width x number of ROIs + """ return self._dataset_file[self.output_struct_name]["filters"][:].transpose([1, 2, 0]) def _trace_extractor_read(self): + """Read the traces from the .mat file and return the traces. + + Returns + ------- + traces : DatasetView + 2-D array: number of frames x number of ROIs + """ return self._dataset_file[self.output_struct_name]["traces"] def _tot_exptime_extractor_read(self): + """Read the total experiment time from the .mat file and return the total experiment time. + + Returns + ------- + tot_exptime : float + The total experiment time in units of seconds. + """ return self._dataset_file[self.output_struct_name]["time"]["totalTime"][0][0] def _summary_image_read(self): + """Read the summary image from the .mat file and return the summary image. + + Returns + ------- + summary_image : numpy.ndarray + The summary image. + """ summary_image = self._dataset_file[self.output_struct_name]["info"]["summary_image"] return np.array(summary_image) def _raw_datafile_read(self): + """Read the raw data file location from the .mat file and return the raw data file location. + + Returns + ------- + raw_datafile : str + The raw data file location. + """ if self._dataset_file[self.output_struct_name].get("file"): charlist = [chr(i) for i in np.squeeze(self._dataset_file[self.output_struct_name]["file"][:])] return "".join(charlist) From bbb7d4518a793b79aca3524ffa2654f23859e58a Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 31 Aug 2023 18:16:14 -0700 Subject: [PATCH 29/39] added docstrings to Tiff --- .../tiffimagingextractors/__init__.py | 26 +++++ .../brukertiffimagingextractor.py | 107 ++++++++++++++---- .../micromanagertiffimagingextractor.py | 65 ++++++++--- .../scanimagetiffimagingextractor.py | 26 ++++- .../tiffimagingextractor.py | 36 +++++- 5 files changed, 213 insertions(+), 47 deletions(-) diff --git a/src/roiextractors/extractors/tiffimagingextractors/__init__.py b/src/roiextractors/extractors/tiffimagingextractors/__init__.py index c7d03795..b8f5cf54 100644 --- a/src/roiextractors/extractors/tiffimagingextractors/__init__.py +++ b/src/roiextractors/extractors/tiffimagingextractors/__init__.py @@ -1,3 +1,29 @@ +"""A collection of ImagingExtractors for TIFF files with various formats. + +Modules +------- +tiffimagingextractor + A ImagingExtractor for TIFF files. +scanimagetiffimagingextractor + Specialized extractor for reading TIFF files produced via ScanImage. +brukertiffimagingextractor + Specialized extractor for reading TIFF files produced via Bruker. +micromanagertiffimagingextractor + Specialized extractor for reading TIFF files produced via Micro-Manager. + +Classes +------- +TiffImagingExtractor + A ImagingExtractor for TIFF files. +ScanImageTiffImagingExtractor + Specialized extractor for reading TIFF files produced via ScanImage. +BrukerTiffMultiPlaneImagingExtractor + Specialized extractor for reading TIFF files produced via Bruker. +BrukerTiffSinglePlaneImagingExtractor + Specialized extractor for reading TIFF files produced via Bruker. +MicroManagerTiffImagingExtractor + Specialized extractor for reading TIFF files produced via Micro-Manager. +""" from .tiffimagingextractor import TiffImagingExtractor from .scanimagetiffimagingextractor import ScanImageTiffImagingExtractor from .brukertiffimagingextractor import BrukerTiffMultiPlaneImagingExtractor, BrukerTiffSinglePlaneImagingExtractor diff --git a/src/roiextractors/extractors/tiffimagingextractors/brukertiffimagingextractor.py b/src/roiextractors/extractors/tiffimagingextractors/brukertiffimagingextractor.py index 0f30f0ff..e307d826 100644 --- a/src/roiextractors/extractors/tiffimagingextractors/brukertiffimagingextractor.py +++ b/src/roiextractors/extractors/tiffimagingextractors/brukertiffimagingextractor.py @@ -1,3 +1,12 @@ +"""ImagingExtractors for the TIFF image format produced by Bruker. + +Classes +------- +BrukerTiffSinglePlaneImagingExtractor + A ImagingExtractor for TIFF files produced by Bruker with only 1 plane. +BrukerTiffMultiPlaneImagingExtractor + A MultiImagingExtractor for TIFF files produced by Bruker with multiple planes. +""" import logging import re from collections import Counter @@ -15,6 +24,7 @@ def filter_read_uic_tag_warnings(record): + """Filter out the warnings from tifffile.read_uic_tag() that are not relevant to the user.""" return not record.msg.startswith("") @@ -22,13 +32,12 @@ def filter_read_uic_tag_warnings(record): def _get_tiff_reader() -> ModuleType: + """Return the tifffile module.""" return get_package(package_name="tifffile", installation_instructions="pip install tifffile") def _determine_frame_rate(element: ElementTree.Element, file_names: Optional[List[str]] = None) -> Union[float, None]: - """ - Determines the frame rate from the difference in relative timestamps of the frame elements. - """ + """Determine the frame rate from the difference in relative timestamps of the frame elements.""" from neuroconv.utils import calculate_regular_series_rate frame_elements = element.findall(".//Frame") @@ -44,9 +53,17 @@ def _determine_frame_rate(element: ElementTree.Element, file_names: Optional[Lis def _determine_imaging_is_volumetric(folder_path: PathType) -> bool: - """ - Determines whether imaging is volumetric based on 'zDevice' configuration value. - The value is expected to be '1' for volumetric and '0' for single plane images. + """Determine whether imaging is volumetric based on 'zDevice' configuration value. + + Parameters + ---------- + folder_path : PathType + The path to the folder that contains the Bruker TIF image files (.ome.tif) and configuration files (.xml, .env). + + Returns + ------- + is_volumetric: bool + True if the imaging is volumetric (multiplane), False otherwise (single plane). """ xml_root = _parse_xml(folder_path=folder_path) z_device_element = xml_root.find(".//PVStateValue[@key='zDevice']") @@ -56,9 +73,7 @@ def _determine_imaging_is_volumetric(folder_path: PathType) -> bool: def _parse_xml(folder_path: PathType) -> ElementTree.Element: - """ - Parses the XML configuration file into element tree and returns the root Element. - """ + """Parse the XML configuration file into element tree and returns the root Element.""" folder_path = Path(folder_path) xml_file_path = folder_path / f"{folder_path.name}.xml" assert xml_file_path.is_file(), f"The XML configuration file is not found at '{folder_path}'." @@ -67,12 +82,29 @@ def _parse_xml(folder_path: PathType) -> ElementTree.Element: class BrukerTiffMultiPlaneImagingExtractor(MultiImagingExtractor): + """A MultiImagingExtractor for TIFF files produced by Bruke with multiple planes. + + This format consists of multiple TIF image files (.ome.tif) and configuration files (.xml, .env). + """ + extractor_name = "BrukerTiffMultiPlaneImaging" is_writable = True mode = "folder" @classmethod def get_streams(cls, folder_path: PathType) -> dict: + """Get the available streams from the Bruker TIF image files (.ome.tif) and configuration files (.xml, .env). + + Parameters + ---------- + folder_path : PathType + The path to the folder that contains the Bruker TIF image files (.ome.tif) and configuration files (.xml, .env). + + Returns + ------- + streams: dict + The dictionary of available streams. + """ natsort = get_package(package_name="natsort", installation_instructions="pip install natsort") xml_root = _parse_xml(folder_path=folder_path) @@ -101,9 +133,7 @@ def __init__( folder_path: PathType, stream_name: Optional[str] = None, ): - """ - The imaging extractor for the Bruker TIF image format. - This format consists of multiple TIF image files (.ome.tif) and configuration files (.xml, .env). + """Create a BrukerTiffMultiPlaneImagingExtractor instance from a folder path that contains the image files. Parameters ---------- @@ -111,6 +141,17 @@ def __init__( The path to the folder that contains the Bruker TIF image files (.ome.tif) and configuration files (.xml, .env). stream_name: str, optional The name of the recording channel (e.g. "Ch2"). + + Raises + ------ + ValueError + If more than one recording stream is detected. + ValueError + If the selected stream is not in the available plane_streams. + AssertionError + If the TIF image files are missing from the folder. + AssertionError + If the imaging is not volumetric. """ self._tifffile = _get_tiff_reader() @@ -159,6 +200,7 @@ def __init__( self._start_frames = [0] * self._num_planes_per_channel_stream self._end_frames = [self._num_frames] * self._num_planes_per_channel_stream + # TODO: fix this method so that it is consistent with base multiimagingextractor method (i.e. num_rows, num_columns) def get_image_size(self) -> Tuple[int, int, int]: return self._image_size @@ -203,12 +245,26 @@ def get_video( class BrukerTiffSinglePlaneImagingExtractor(MultiImagingExtractor): + """A MultiImagingExtractor for TIFF files produced by Bruker with only 1 plane.""" + extractor_name = "BrukerTiffSinglePlaneImaging" is_writable = True mode = "folder" @classmethod def get_streams(cls, folder_path: PathType) -> dict: + """Get the available streams from the Bruker TIF image files (.ome.tif) and configuration files (.xml, .env). + + Parameters + ---------- + folder_path : PathType + The path to the folder that contains the Bruker TIF image files (.ome.tif) and configuration files (.xml, .env). + + Returns + ------- + streams: dict + The dictionary of available streams. + """ natsort = get_package(package_name="natsort", installation_instructions="pip install natsort") xml_root = _parse_xml(folder_path=folder_path) channel_names = [file.attrib["channelName"] for file in xml_root.findall(".//File")] @@ -217,9 +273,7 @@ def get_streams(cls, folder_path: PathType) -> dict: return streams def __init__(self, folder_path: PathType, stream_name: Optional[str] = None): - """ - The imaging extractor for the Bruker TIF image format. - This format consists of multiple TIF image files (.ome.tif) and configuration files (.xml, .env). + """Create a BrukerTiffSinglePlaneImagingExtractor instance from a folder path that contains the image files. Parameters ---------- @@ -286,9 +340,12 @@ def __init__(self, folder_path: PathType, stream_name: Optional[str] = None): super().__init__(imaging_extractors=imaging_extractors) def _get_xml_metadata(self) -> Dict[str, Union[str, List[Dict[str, str]]]]: - """ - Parses the metadata in the root element that are under "PVStateValue" tag into - a dictionary. + """Parse the metadata in the root element that are under "PVStateValue" tag into a dictionary. + + Returns + ------- + xml_metadata: dict + The dictionary of metadata extracted from the XML file. """ xml_metadata = dict() xml_metadata.update(**self._xml_root.attrib) @@ -322,7 +379,7 @@ def _get_xml_metadata(self) -> Dict[str, Union[str, List[Dict[str, str]]]]: return xml_metadata def _check_consistency_between_imaging_extractors(self): - """Overrides the parent class method as none of the properties that are checked are from the sub-imaging extractors.""" + """Override the parent class method as none of the properties that are checked are from the sub-imaging extractors.""" return True def get_image_size(self) -> Tuple[int, int]: @@ -342,6 +399,13 @@ def get_dtype(self) -> DtypeType: class _BrukerTiffSinglePlaneImagingExtractor(ImagingExtractor): + """A private ImagingExtractor for TIFF files produced by Bruker with only 1 plane. + + The private imaging extractor for OME-TIF image format produced by Bruker, + which defines the get_video() method to return the requested frames from a given file. + This extractor is not meant to be used as a standalone ImagingExtractor. + """ + extractor_name = "_BrukerTiffSinglePlaneImaging" is_writable = True mode = "file" @@ -351,10 +415,7 @@ class _BrukerTiffSinglePlaneImagingExtractor(ImagingExtractor): DATA_TYPE_ERROR = "The {}Extractor does not support retrieving the data type." def __init__(self, file_path: PathType): - """ - The private imaging extractor for OME-TIF image format produced by Bruker, - which defines the get_video() method to return the requested frames from a given file. - This extractor is not meant to be used as a standalone ImagingExtractor. + """Create a _BrukerTiffSinglePlaneImagingExtractor instance from a TIFF image file (.ome.tif). Parameters ---------- diff --git a/src/roiextractors/extractors/tiffimagingextractors/micromanagertiffimagingextractor.py b/src/roiextractors/extractors/tiffimagingextractors/micromanagertiffimagingextractor.py index 7ab1db11..21d79282 100644 --- a/src/roiextractors/extractors/tiffimagingextractors/micromanagertiffimagingextractor.py +++ b/src/roiextractors/extractors/tiffimagingextractors/micromanagertiffimagingextractor.py @@ -1,3 +1,11 @@ +"""A ImagingExtractor for TIFF files produced by Micro-Manager. + +Classes +------- +MicroManagerTiffImagingExtractor + A ImagingExtractor for TIFF files produced by Micro-Manager. +""" + import json import logging import re @@ -27,14 +35,17 @@ def _get_tiff_reader() -> ModuleType: class MicroManagerTiffImagingExtractor(MultiImagingExtractor): + """Specialized extractor for reading TIFF files produced via Micro-Manager. + + The image file stacks are saved into multipage TIF files in OME-TIFF format (.ome.tif files), + each of which are up to around 4GB in size. + The 'DisplaySettings' JSON file contains the properties of Micro-Manager. + """ + extractor_name = "MicroManagerTiffImaging" def __init__(self, folder_path: PathType): - """ - The imaging extractor for the Micro-Manager TIF image format. - The image file stacks are saved into multipage TIF files in OME-TIFF format (.ome.tif files), - each of which are up to around 4GB in size. - The 'DisplaySettings' JSON file contains the properties of Micro-Manager. + """Create a MicroManagerTiffImagingExtractor instance from a folder path that contains the image files. Parameters ---------- @@ -93,8 +104,12 @@ def __init__(self, folder_path: PathType): super().__init__(imaging_extractors=imaging_extractors) def _load_settings_json(self) -> Dict[str, Dict[str, str]]: - """ - Loads the 'DisplaySettings' JSON file. + """Load the 'DisplaySettings' JSON file. + + Returns + ------- + settings: Dict[str, Dict[str, str]] + The dictionary that contains the properties of Micro-Manager. """ file_name = "DisplaySettings.json" settings_json_file_path = self.folder_path / file_name @@ -106,17 +121,29 @@ def _load_settings_json(self) -> Dict[str, Dict[str, str]]: return settings["map"] def _get_ome_xml_root(self) -> ElementTree: - """ - Parses the OME-XML configuration from string format into element tree and returns the root of this tree. + """Parse the OME-XML configuration from string format into element tree and returns the root of this tree. + + Returns + ------- + root: ElementTree + The root of the element tree that contains the OME-XML configuration. """ ome_metadata_element = ElementTree.fromstring(self._ome_metadata) tree = ElementTree.ElementTree(ome_metadata_element) return tree.getroot() def _check_missing_files_in_folder(self, expected_list_of_files): - """ - Checks the presence of each TIF file that is expected to be found in the folder. - Raises an error when the files are not found with the name of the missing files. + """Check the presence of each TIF file that is expected to be found in the folder. + + Parameters + ---------- + expected_list_of_files: list + The list of file names that are expected to be found in the folder. + + Raises + ------ + AssertionError + Raises an error when the files are not found with the name of the missing files. """ missing_files = [ file_name for file_name in expected_list_of_files if self.folder_path / file_name not in self._ome_tif_files @@ -126,7 +153,7 @@ def _check_missing_files_in_folder(self, expected_list_of_files): ), f"Some of the TIF image files at '{self.folder_path}' are missing. The list of files that are missing: {missing_files}" def _check_consistency_between_imaging_extractors(self): - """Overrides the parent class method as none of the properties that are checked are from the sub-imaging extractors.""" + """Override the parent class method as none of the properties that are checked are from the sub-imaging extractors.""" return True def get_image_size(self) -> Tuple[int, int]: @@ -149,6 +176,13 @@ def get_dtype(self) -> DtypeType: class _MicroManagerTiffImagingExtractor(ImagingExtractor): + """Private imaging extractor for OME-TIF image format produced by Micro-Manager. + + The private imaging extractor for OME-TIF image format produced by Micro-Manager, + which defines the get_video() method to return the requested frames from a given file. + This extractor is not meant to be used as a standalone ImagingExtractor. + """ + extractor_name = "_MicroManagerTiffImaging" is_writable = True mode = "file" @@ -158,10 +192,7 @@ class _MicroManagerTiffImagingExtractor(ImagingExtractor): DATA_TYPE_ERROR = "The {}Extractor does not support retrieving the data type." def __init__(self, file_path: PathType): - """ - The private imaging extractor for OME-TIF image format produced by Micro-Manager, - which defines the get_video() method to return the requested frames from a given file. - This extractor is not meant to be used as a standalone ImagingExtractor. + """Create a _MicroManagerTiffImagingExtractor instance from a TIFF image file (.ome.tif). Parameters ---------- diff --git a/src/roiextractors/extractors/tiffimagingextractors/scanimagetiffimagingextractor.py b/src/roiextractors/extractors/tiffimagingextractors/scanimagetiffimagingextractor.py index 05f8c723..98c21323 100644 --- a/src/roiextractors/extractors/tiffimagingextractors/scanimagetiffimagingextractor.py +++ b/src/roiextractors/extractors/tiffimagingextractors/scanimagetiffimagingextractor.py @@ -1,4 +1,10 @@ -"""Specialized extractor for reading TIFF files produced via ScanImage.""" +"""Specialized extractor for reading TIFF files produced via ScanImage. + +Classes +------- +ScanImageTiffImagingExtractor + Specialized extractor for reading TIFF files produced via ScanImage. +""" from pathlib import Path from typing import Optional, Tuple from warnings import warn @@ -27,8 +33,7 @@ def __init__( file_path: PathType, sampling_frequency: FloatType, ): - """ - Specialized extractor for reading TIFF files produced via ScanImage. + """Create a ScanImageTiffImagingExtractor instance from a TIFF file produced by ScanImage. This extractor allows for lazy accessing of slices, unlike :py:class:`~roiextractors.extractors.tiffimagingextractors.TiffImagingExtractor`. @@ -82,11 +87,20 @@ def get_frames(self, frame_idxs: ArrayType, channel: int = 0) -> np.ndarray: frames = frames.squeeze() return frames + # Data accessed through an open ScanImageTiffReader io gets scrambled if there are multiple calls. + # Thus, open fresh io in context each time something is needed. def _get_single_frame(self, idx: int) -> np.ndarray: - """ - Data accessed through an open ScanImageTiffReader io gets scrambled if there are multiple calls. + """Get a single frame of data from the TIFF file. + + Parameters + ---------- + idx : int + The index of the frame to retrieve. - Thus, open fresh io in context each time something is needed. + Returns + ------- + frame: numpy.ndarray + The frame of data. """ ScanImageTiffReader = _get_scanimage_reader() diff --git a/src/roiextractors/extractors/tiffimagingextractors/tiffimagingextractor.py b/src/roiextractors/extractors/tiffimagingextractors/tiffimagingextractor.py index 635002d8..1cd21cc7 100644 --- a/src/roiextractors/extractors/tiffimagingextractors/tiffimagingextractor.py +++ b/src/roiextractors/extractors/tiffimagingextractors/tiffimagingextractor.py @@ -1,3 +1,10 @@ +"""A TIFF imaging extractor for TIFF files. + +Classes +------- +TiffImagingExtractor + A TIFF imaging extractor for TIFF files. +""" from pathlib import Path from typing import Optional from warnings import warn @@ -16,11 +23,22 @@ class TiffImagingExtractor(ImagingExtractor): + """A ImagingExtractor for TIFF files.""" + extractor_name = "TiffImaging" is_writable = True mode = "file" def __init__(self, file_path: PathType, sampling_frequency: FloatType): + """Create a TiffImagingExtractor instance from a TIFF file. + + Parameters + ---------- + file_path : PathType + Path to the TIFF file. + sampling_frequency : float + The frequency at which the frames were sampled, in Hz. + """ tifffile = get_package(package_name="tifffile") super().__init__() @@ -29,7 +47,7 @@ def __init__(self, file_path: PathType, sampling_frequency: FloatType): if self.file_path.suffix not in [".tiff", ".tif", ".TIFF", ".TIF"]: warn( "File suffix ({self.file_path.suffix}) is not one of .tiff, .tif, .TIFF, or .TIF! " - "The TiffImagingExtracto may not be appropriate." + "The TiffImagingExtractor may not be appropriate." ) with tifffile.TiffFile(self.file_path) as tif: @@ -80,6 +98,22 @@ def get_channel_names(self): @staticmethod def write_imaging(imaging, save_path, overwrite: bool = False, chunk_size=None, verbose=True): + """Write a TIFF file from an ImagingExtractor. + + Parameters + ---------- + imaging : ImagingExtractor + The ImagingExtractor to be written to a TIFF file. + save_path : str or PathType + The path to save the TIFF file. + overwrite : bool + If True, will overwrite the file if it exists. Otherwise will raise an error if the file exists. + chunk_size : int or None + If None, will write the entire video to a single TIFF file. Otherwise will write the video + in chunk_size frames at a time. + verbose : bool + If True, will print progress bar. + """ tifffile = get_package(package_name="tifffile") save_path = Path(save_path) From b87b071cdd2233aadc532564b7ddf741f7ee5506 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 5 Sep 2023 10:37:30 -0700 Subject: [PATCH 30/39] added docstrings to extraction_tools --- src/roiextractors/extraction_tools.py | 219 ++++++++++++++++++++++---- 1 file changed, 191 insertions(+), 28 deletions(-) diff --git a/src/roiextractors/extraction_tools.py b/src/roiextractors/extraction_tools.py index ded150a7..a41a9040 100644 --- a/src/roiextractors/extraction_tools.py +++ b/src/roiextractors/extraction_tools.py @@ -70,14 +70,22 @@ class VideoStructure: The role of the data class is to ensure consistency in naming and provide some initial consistency checks to ensure the validity of the sturcture. - Attributes: - num_rows (int): The number of rows of each frame as a matrix. - num_columns (int): The number of columns of each frame as a matrix. - num_channels (int): The number of chanenls (1 for gray, 3 for colors). - rows_axis (int): The axis or dimension corresponding to the rows. - columns_axis (int): The axis or dimension corresponding to the columns. - channels_axis (int): The axis or dimension corresponding to the channels. - frame_axis (int): The axis or dimension corresponding to the frames in the video. + Attributes + ---------- + num_rows : int + The number of rows of each frame as a matrix. + num_columns : int + The number of columns of each frame as a matrix. + num_channels : int + The number of channels (1 for grayscale, 3 for color). + rows_axis : int + The axis or dimension corresponding to the rows. + columns_axis : int + The axis or dimension corresponding to the columns. + channels_axis : int + The axis or dimension corresponding to the channels. + frame_axis : int + The axis or dimension corresponding to the frames in the video. As an example if you wanted to build the structure for a video with gray (n_channels=1) frames of 10 x 5 where the video is to have the following shape (num_frames, num_rows, num_columns, num_channels) you @@ -112,11 +120,13 @@ class VideoStructure: frame_axis: int def __post_init__(self) -> None: + """Validate the structure of the video and initialize the shape of the frame.""" self._validate_video_structure() self._initialize_frame_shape() self.number_of_pixels_per_frame = np.prod(self.frame_shape) def _initialize_frame_shape(self) -> None: + """Initialize the shape of the frame.""" self.frame_shape = [None, None, None, None] self.frame_shape[self.rows_axis] = self.num_rows self.frame_shape[self.columns_axis] = self.num_columns @@ -125,6 +135,7 @@ def _initialize_frame_shape(self) -> None: self.frame_shape = tuple(self.frame_shape) def _validate_video_structure(self) -> None: + """Validate the structure of the video.""" exception_message = ( "Invalid structure: " f"{self.__repr__()}, " @@ -141,6 +152,23 @@ def _validate_video_structure(self) -> None: raise ValueError(exception_message) def build_video_shape(self, n_frames: int) -> Tuple[int, int, int, int]: + """Build the shape of the video from class attributes. + + Parameters + ---------- + n_frames : int + The number of frames in the video. + + Returns + ------- + Tuple[int, int, int, int] + The shape of the video. + + Notes + ----- + The class attributes frame_axis, rows_axis, columns_axis and channels_axis are used to determine the order of the + dimensions in the returned tuple. + """ video_shape = [None] * 4 video_shape[self.frame_axis] = n_frames video_shape[self.rows_axis] = self.num_rows @@ -150,23 +178,21 @@ def build_video_shape(self, n_frames: int) -> Tuple[int, int, int, int]: return tuple(video_shape) def transform_video_to_canonical_form(self, video: np.ndarray) -> np.ndarray: - """Transform a video with the structure in this class to the canonical internal format of - roiextractors (num_frames, num_rows, num_columns, num_channels) + """Transform a video to the canonical internal format of roiextractors (num_frames, num_rows, num_columns, num_channels). - The function supports either Parameters ---------- - video : np.ndarray + video : numpy.ndarray The video to be transformed Returns ------- - np.ndarray + numpy.ndarray The reshaped video Raises ------ KeyError - + If the video is not in a format that can be transformed. """ canonical_shape = (self.frame_axis, self.rows_axis, self.columns_axis, self.channels_axis) if isinstance(video, (h5py.Dataset, zarr.core.Array)): @@ -225,7 +251,6 @@ def read_numpy_memmap_video( video_memap: np.array A numpy memmap pointing to the video. """ - file_size_bytes = Path(file_path).stat().st_size pixels_per_frame = video_structure.number_of_pixels_per_frame @@ -242,9 +267,18 @@ def read_numpy_memmap_video( def _pixel_mask_extractor(image_mask_, _roi_ids): - """An alternative data format for storage of image masks which relies on the sparsity of the images. + """Convert image mask to pixel mask. + + Pixel masks are an alternative data format for storage of image masks which relies on the sparsity of the images. The location and weight of each non-zero pixel is stored for each mask. + Parameters + ---------- + image_mask_: numpy.ndarray + Dense representation of the ROIs with shape (number_of_rows, number_of_columns, number_of_rois). + _roi_ids: list + List of roi ids with length number_of_rois. + Returns ------- pixel_masks: list @@ -262,19 +296,21 @@ def _pixel_mask_extractor(image_mask_, _roi_ids): def _image_mask_extractor(pixel_mask, _roi_ids, image_shape): - """ - Converts a pixel mask to image mask + """Convert a pixel mask to image mask. Parameters ---------- pixel_mask: list list of pixel masks (no pixels X 3) _roi_ids: list + list of roi ids with length number_of_rois image_shape: array_like + shape of the image (number_of_rows, number_of_columns) Returns ------- image_mask: np.ndarray + Dense representation of the ROIs with shape (number_of_rows, number_of_columns, number_of_rois). """ image_mask = np.zeros(list(image_shape) + [len(_roi_ids)]) for no, rois in enumerate(_roi_ids): @@ -284,6 +320,18 @@ def _image_mask_extractor(pixel_mask, _roi_ids, image_shape): def get_video_shape(video): + """Get the shape of a video (num_channels, num_frames, size_x, size_y). + + Parameters + ---------- + video: numpy.ndarray + The video to get the shape of. + + Returns + ------- + video_shape: tuple + The shape of the video (num_channels, num_frames, size_x, size_y). + """ if len(video.shape) == 3: # 1 channel num_channels = 1 @@ -294,9 +342,25 @@ def get_video_shape(video): def check_get_frames_args(func): - """ + """Check the arguments of the get_frames function. + This decorator allows the get_frames function to be queried with either an integer, slice or an array and handles a common return. [I think that np.take can be used instead of this] + + Parameters + ---------- + func: function + The get_frames function. + + Returns + ------- + corrected_args: function + The get_frames function with corrected arguments. + + Raises + ------ + AssertionError + If 'frame_idxs' exceed the number of frames. """ @wraps(func) @@ -318,6 +382,29 @@ def corrected_args(imaging, frame_idxs, channel=0): def _cast_start_end_frame(start_frame, end_frame): + """Cast start and end frame to int or None. + + Parameters + ---------- + start_frame: int, float, None + The start frame. + end_frame: int, float, None + The end frame. + + Returns + ------- + start_frame: int, None + The start frame. + end_frame: int, None + The end frame. + + Raises + ------ + ValueError + If start_frame is not an int, float or None. + ValueError + If end_frame is not an int, float or None. + """ if isinstance(start_frame, float): start_frame = int(start_frame) elif isinstance(start_frame, (int, np.integer, type(None))): @@ -337,6 +424,31 @@ def _cast_start_end_frame(start_frame, end_frame): def check_get_videos_args(func): + """Check the arguments of the get_videos function. + + This decorator allows the get_videos function to be queried with either + an integer or slice and handles a common return. + + Parameters + ---------- + func: function + The get_videos function. + + Returns + ------- + corrected_args: function + The get_videos function with corrected arguments. + + Raises + ------ + AssertionError + If 'start_frame' exceeds the number of frames. + AssertionError + If 'end_frame' exceeds the number of frames. + AssertionError + If 'start_frame' is greater than 'end_frame'. + """ + @wraps(func) def corrected_args(imaging, start_frame=None, end_frame=None, channel=0): if start_frame is not None: @@ -374,12 +486,12 @@ def write_to_h5_dataset_format( chunk_mb=1000, verbose=False, ): - """Saves the video of an imaging extractor in an h5 dataset. + """Save the video of an imaging extractor in an h5 dataset. Parameters ---------- imaging: ImagingExtractor - The imaging extractor object to be saved in the .h5 filr + The imaging extractor object to be saved in the .h5 file dataset_path: str Path to dataset in h5 file (e.g. '/dataset') save_path: str @@ -396,6 +508,18 @@ def write_to_h5_dataset_format( Chunk size in Mb (default 1000Mb) verbose: bool If True, output is verbose (when chunks are used) + + Returns + ------- + save_path: str + The path to the file. + + Raises + ------ + AssertionError + If h5py is not installed. + AssertionError + If neither 'save_path' nor 'file_handle' are given. """ assert HAVE_H5, "To write to h5 you need to install h5py: pip install h5py" assert save_path is not None or file_handle is not None, "Provide 'save_path' or 'file handle'" @@ -461,6 +585,20 @@ def write_to_h5_dataset_format( # TODO will be moved eventually, but for now it's very handy :) def show_video(imaging, ax=None): + """Show video as animation. + + Parameters + ---------- + imaging: ImagingExtractor + The imaging extractor object to be saved in the .h5 file + ax: matplotlib axis + Axis to plot the video. If None, a new axis is created. + + Returns + ------- + anim: matplotlib.animation.FuncAnimation + Animation of the video. + """ import matplotlib.pyplot as plt import matplotlib.animation as animation @@ -487,9 +625,25 @@ def animate_func(i, imaging, im, ax): def check_keys(dict): - """ - checks if entries in dictionary are mat-objects. If yes - todict is called to change them to nested dictionaries + """Check keys of dictionary for mat-objects. + + Checks if entries in dictionary are mat-objects. If yes + todict is called to change them to nested dictionaries. + + Parameters + ---------- + dict: dict + Dictionary to check. + + Returns + ------- + dict: dict + Dictionary with mat-objects converted to nested dictionaries. + + Raises + ------ + AssertionError + If scipy is not installed. """ assert HAVE_Scipy, "To write to h5 you need to install scipy: pip install scipy" for key in dict: @@ -499,8 +653,17 @@ def check_keys(dict): def todict(matobj): - """ - A recursive function which constructs from matobjects nested dictionaries + """Recursively construct nested dictionaries from matobjects. + + Parameters + ---------- + matobj: mat_struct + Matlab object to convert to nested dictionary. + + Returns + ------- + dict: dict + Dictionary with mat-objects converted to nested dictionaries. """ dict = {} for strg in matobj._fieldnames: @@ -517,8 +680,7 @@ def get_package( installation_instructions: Optional[str] = None, excluded_platforms_and_python_versions: Optional[Dict[str, List[str]]] = None, ) -> ModuleType: - """ - Check if package is installed and return module if so. + """Check if package is installed and return module if so. Otherwise, raise informative error describing how to perform the installation. Inspired by https://docs.python.org/3/library/importlib.html#checking-if-a-module-can-be-imported. @@ -542,6 +704,7 @@ def get_package( Raises ------ ModuleNotFoundError + If the package is not installed. """ installation_instructions = installation_instructions or f"pip install {package_name}" excluded_platforms_and_python_versions = excluded_platforms_and_python_versions or dict() From 1167d135ce946ddbfb2418cc393bd2c8193b466d Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 5 Sep 2023 13:45:38 -0700 Subject: [PATCH 31/39] fixed docstring checker to ignore dataclass methods --- src/roiextractors/extraction_tools.py | 8 +++++++- tests/test_docstrings.py | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/roiextractors/extraction_tools.py b/src/roiextractors/extraction_tools.py index a41a9040..1ceed3d4 100644 --- a/src/roiextractors/extraction_tools.py +++ b/src/roiextractors/extraction_tools.py @@ -1,3 +1,10 @@ +"""Various tools for extraction of ROIs from imaging data. + +Classes +------- +VideoStructure + A data class for specifying the structure of a video. +""" import sys import importlib.util from functools import wraps @@ -108,7 +115,6 @@ class VideoStructure: channels_axis=channels_axis, frame_axis=frame_axis, ) - """ num_rows: int diff --git a/tests/test_docstrings.py b/tests/test_docstrings.py index 7e43ab8b..4a82fa93 100644 --- a/tests/test_docstrings.py +++ b/tests/test_docstrings.py @@ -50,6 +50,8 @@ def test_has_docstring(obj): msg = f"{obj.__name__} has no docstring." else: msg = f"{obj.__module__}.{obj.__qualname__} has no docstring." + if "__create_fn__" in msg: + return # skip dataclass functions created by __create_fn__ assert doc is not None, msg From 78562a0fb1ce8ead10281fd36825e76814ed7fce Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 5 Sep 2023 13:53:59 -0700 Subject: [PATCH 32/39] added docstrings to package __init__s --- src/roiextractors/__init__.py | 1 + src/roiextractors/extractors/__init__.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/roiextractors/__init__.py b/src/roiextractors/__init__.py index eadef868..46c64c68 100644 --- a/src/roiextractors/__init__.py +++ b/src/roiextractors/__init__.py @@ -1,3 +1,4 @@ +"""Python-based module for extracting from, converting between, and handling recorded and optical imaging data from several file formats.""" # Keeping __version__ accessible only to maintain backcompatability. # Modern appraoch (Python >= 3.8) is to use importlib try: diff --git a/src/roiextractors/extractors/__init__.py b/src/roiextractors/extractors/__init__.py index e69de29b..25e0445d 100644 --- a/src/roiextractors/extractors/__init__.py +++ b/src/roiextractors/extractors/__init__.py @@ -0,0 +1 @@ +"""All specialized ImagingExtractors and SegmentationExtractors are defined here.""" From 89b146e58bd8852951c597228404d3bff2624e77 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 5 Sep 2023 14:25:42 -0700 Subject: [PATCH 33/39] fixed bug in docstring test --- tests/test_docstrings.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_docstrings.py b/tests/test_docstrings.py index 4a82fa93..26588dc3 100644 --- a/tests/test_docstrings.py +++ b/tests/test_docstrings.py @@ -7,14 +7,16 @@ def traverse_class(cls, objs): """Traverse a class and its methods and append them to objs.""" - for name, obj in inspect.getmembers(cls, inspect.isfunction or inspect.ismethod): + predicate = lambda x: inspect.isfunction(x) or inspect.ismethod(x) + for name, obj in inspect.getmembers(cls, predicate=predicate): objs.append(obj) def traverse_module(module, objs): """Traverse all classes and functions in a module and append them to objs.""" objs.append(module) - for name, obj in inspect.getmembers(module, inspect.isclass or inspect.isfunction or inspect.ismethod): + predicate = lambda x: inspect.isclass(x) or inspect.isfunction(x) or inspect.ismethod(x) + for name, obj in inspect.getmembers(module, predicate=predicate): parent_package = obj.__module__.split(".")[0] if parent_package != "roiextractors": # avoid traversing external dependencies continue @@ -39,7 +41,6 @@ def traverse_package(package, objs): objs = [] traverse_package(roiextractors, objs) -print(objs) @pytest.mark.parametrize("obj", objs) From 6a1a83f950ee2992fea3f0a6fc8fe1386909733a Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 5 Sep 2023 14:32:31 -0700 Subject: [PATCH 34/39] added docstrings to toy_example --- .../example_datasets/__init__.py | 12 ++++++ .../example_datasets/toy_example.py | 43 ++++++++++++++++--- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/src/roiextractors/example_datasets/__init__.py b/src/roiextractors/example_datasets/__init__.py index 135ecd47..a1951668 100644 --- a/src/roiextractors/example_datasets/__init__.py +++ b/src/roiextractors/example_datasets/__init__.py @@ -1 +1,13 @@ +"""Toy example ImagingExtractor and SegmentationExtractor for testing. + +Modules +------- +toy_example + Create a toy example of an ImagingExtractor and a SegmentationExtractor. + +Functions +--------- +toy_example + Create a toy example of an ImagingExtractor and a SegmentationExtractor. +""" from .toy_example import toy_example diff --git a/src/roiextractors/example_datasets/toy_example.py b/src/roiextractors/example_datasets/toy_example.py index 7d97f4fc..968c77cc 100644 --- a/src/roiextractors/example_datasets/toy_example.py +++ b/src/roiextractors/example_datasets/toy_example.py @@ -1,3 +1,10 @@ +"""Toy example ImagingExtractor and SegmentationExtractor for testing. + +Functions +--------- +toy_example + Create a toy example of an ImagingExtractor and a SegmentationExtractor. +""" import numpy as np from ..extractors.numpyextractors import ( @@ -7,10 +14,39 @@ def _gaussian(x, mu, sigma): + """Compute classical gaussian with parameters x, mu, sigma.""" return 1 / np.sqrt(2 * np.pi * sigma) * np.exp(-((x - mu) ** 2) / sigma) def _generate_rois(num_units=10, size_x=100, size_y=100, roi_size=4, min_dist=5, mode="uniform"): + """Generate ROIs with given parameters. + + Parameters + ---------- + num_units: int + Number of ROIs + size_x: int + Size of x dimension (pixels) + size_y: int + Size of y dimension (pixels) + roi_size: int + Siz of ROI in x and y dimension (pixels) + min_dist: int + Minimum distance between ROI centers (pixels) + mode: str + 'uniform' or 'gaussian'. + If 'uniform', ROI values are uniform and equal to 1. + If 'gaussian', ROI values are gaussian modulated + + Returns + ------- + roi_pixels: list + List of pixel coordinates for each ROI + image: np.ndarray + Image with ROIs + means: list + List of mean coordinates for each ROI + """ image = np.zeros((size_x, size_y)) max_iter = 1000 @@ -73,8 +109,7 @@ def toy_example( decay_time=0.5, noise_std=0.05, ): - """ - Create a toy example of an ImagingExtractor and a SegmentationExtractor. + """Create a toy example of an ImagingExtractor and a SegmentationExtractor. Parameters ---------- @@ -87,7 +122,7 @@ def toy_example( size_y: int Size of y dimension (pixels) roi_size: int - Siz of ROI in x and y dimension (pixels) + Size of ROI in x and y dimension (pixels) min_dist: int Minimum distance between ROI centers (pixels) mode: str @@ -107,9 +142,7 @@ def toy_example( The output imaging extractor seg: NumpySegmentationExtractor The output segmentation extractor - """ - # generate ROIs num_rois = int(num_rois) roi_pixels, im, means = _generate_rois( From c3529d68fd6453d6b9dda772892cf3951332365e Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 5 Sep 2023 14:36:53 -0700 Subject: [PATCH 35/39] added docstrings to helper functions in tiffimagingextractors --- .../tiffimagingextractors/micromanagertiffimagingextractor.py | 2 ++ .../tiffimagingextractors/scanimagetiffimagingextractor.py | 1 + 2 files changed, 3 insertions(+) diff --git a/src/roiextractors/extractors/tiffimagingextractors/micromanagertiffimagingextractor.py b/src/roiextractors/extractors/tiffimagingextractors/micromanagertiffimagingextractor.py index 21d79282..bfc64d6c 100644 --- a/src/roiextractors/extractors/tiffimagingextractors/micromanagertiffimagingextractor.py +++ b/src/roiextractors/extractors/tiffimagingextractors/micromanagertiffimagingextractor.py @@ -24,6 +24,7 @@ def filter_tiff_tag_warnings(record): + """Filter out the warning messages from tifffile package.""" return not record.msg.startswith("") @@ -31,6 +32,7 @@ def filter_tiff_tag_warnings(record): def _get_tiff_reader() -> ModuleType: + """Import the tifffile package and return the module.""" return get_package(package_name="tifffile", installation_instructions="pip install tifffile") diff --git a/src/roiextractors/extractors/tiffimagingextractors/scanimagetiffimagingextractor.py b/src/roiextractors/extractors/tiffimagingextractors/scanimagetiffimagingextractor.py index 98c21323..83a51916 100644 --- a/src/roiextractors/extractors/tiffimagingextractors/scanimagetiffimagingextractor.py +++ b/src/roiextractors/extractors/tiffimagingextractors/scanimagetiffimagingextractor.py @@ -16,6 +16,7 @@ def _get_scanimage_reader() -> type: + """Import the scanimage-tiff-reader package and return the ScanImageTiffReader class.""" return get_package( package_name="ScanImageTiffReader", installation_instructions="pip install scanimage-tiff-reader" ).ScanImageTiffReader From 295f0c189fe9fb22a2e5e5e822d96fc9acd9f8d8 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 5 Sep 2023 14:47:50 -0700 Subject: [PATCH 36/39] added docstrings to testing --- src/roiextractors/testing.py | 85 ++++++++++++++++++++++++------------ 1 file changed, 58 insertions(+), 27 deletions(-) diff --git a/src/roiextractors/testing.py b/src/roiextractors/testing.py index 8f4ef295..8d30a9cb 100644 --- a/src/roiextractors/testing.py +++ b/src/roiextractors/testing.py @@ -17,6 +17,20 @@ def generate_dummy_video(size: Tuple[int], dtype: DtypeType = "uint16"): + """Generate a dummy video of a given size and dtype. + + Parameters + ---------- + size : Tuple[int] + Size of the video to generate. + dtype : DtypeType, optional + Dtype of the video to generate, by default "uint16". + + Returns + ------- + video : np.ndarray + A dummy video of the given size and dtype. + """ dtype = np.dtype(dtype) number_of_bytes = dtype.itemsize @@ -39,6 +53,30 @@ def generate_dummy_imaging_extractor( sampling_frequency: float = 30, dtype: DtypeType = "uint16", ): + """Generate a dummy imaging extractor for testing. + + The imaging extractor is built by feeding random data into the `NumpyImagingExtractor`. + + Parameters + ---------- + num_frames : int, optional + number of frames in the video, by default 30. + num_rows : int, optional + number of rows in the video, by default 10. + num_columns : int, optional + number of columns in the video, by default 10. + num_channels : int, optional + number of channels in the video, by default 1. + sampling_frequency : float, optional + sampling frequency of the video, by default 30. + dtype : DtypeType, optional + dtype of the video, by default "uint16". + + Returns + ------- + ImagingExtractor + An imaging extractor with random data fed into `NumpyImagingExtractor`. + """ channel_names = [f"channel_num_{num}" for num in range(num_channels)] size = (num_frames, num_rows, num_columns, num_channels) @@ -64,13 +102,10 @@ def generate_dummy_segmentation_extractor( has_neuropil_signal: bool = True, rejected_list: Optional[list] = None, ) -> SegmentationExtractor: - """ - A dummy segmentation extractor for testing. The segmentation extractor is built by feeding random data into the - `NumpySegmentationExtractor`. + """Generate a dummy segmentation extractor for testing. - Note that this dummy example is meant to be a mock object with the right shape, structure and objects but does not - contain meaningful content. That is, the image masks matrices are not plausible image mask for a roi, the raw signal - is not a meaningful biological signal and is not related appropriately to the deconvolved signal , etc. + The segmentation extractor is built by feeding random data into the + `NumpySegmentationExtractor`. Parameters ---------- @@ -101,8 +136,13 @@ def generate_dummy_segmentation_extractor( ------- SegmentationExtractor A segmentation extractor with random data fed into `NumpySegmentationExtractor` - """ + Notes + ----- + Note that this dummy example is meant to be a mock object with the right shape, structure and objects but does not + contain meaningful content. That is, the image masks matrices are not plausible image mask for a roi, the raw signal + is not a meaningful biological signal and is not related appropriately to the deconvolved signal , etc. + """ # Create dummy image masks image_masks = np.random.rand(num_rows, num_columns, num_rois) movie_dims = (num_rows, num_columns) @@ -150,6 +190,7 @@ def generate_dummy_segmentation_extractor( def _assert_iterable_shape(iterable, shape): + """Assert that the iterable has the given shape. If the iterable is a numpy array, the shape is checked directly.""" ar = iterable if isinstance(iterable, np.ndarray) else np.array(iterable) for ar_shape, given_shape in zip(ar.shape, shape): if isinstance(given_shape, int): @@ -157,6 +198,7 @@ def _assert_iterable_shape(iterable, shape): def _assert_iterable_shape_max(iterable, shape_max): + """Assert that the iterable has a shape less than or equal to the given maximum shape.""" ar = iterable if isinstance(iterable, np.ndarray) else np.array(iterable) for ar_shape, given_shape in zip(ar.shape, shape_max): if isinstance(given_shape, int): @@ -164,6 +206,7 @@ def _assert_iterable_shape_max(iterable, shape_max): def _assert_iterable_element_dtypes(iterable, dtypes): + """Assert that the iterable has elements of the given dtypes.""" if isinstance(iterable, Iterable) and not isinstance(iterable, str): for iter in iterable: _assert_iterable_element_dtypes(iter, dtypes) @@ -172,6 +215,7 @@ def _assert_iterable_element_dtypes(iterable, dtypes): def _assert_iterable_complete(iterable, dtypes=None, element_dtypes=None, shape=None, shape_max=None): + """Assert that the iterable is complete, i.e. it is not None and has the given dtypes, element_dtypes, shape and shape_max.""" assert isinstance(iterable, dtypes), f"iterable {type(iterable)} is none of the types {dtypes}" if not isinstance(iterable, NoneType): if shape is not None: @@ -185,6 +229,7 @@ def _assert_iterable_complete(iterable, dtypes=None, element_dtypes=None, shape= def check_segmentations_equal( segmentation_extractor1: SegmentationExtractor, segmentation_extractor2: SegmentationExtractor ): + """Check that two segmentation extractors have equal fields.""" check_segmentation_return_types(segmentation_extractor1) check_segmentation_return_types(segmentation_extractor2) # assert equality: @@ -224,9 +269,7 @@ def check_segmentations_images( segmentation_extractor1: SegmentationExtractor, segmentation_extractor2: SegmentationExtractor, ): - """ - Check that the segmentation images are equal for the given segmentation extractors. - """ + """Check that the segmentation images are equal for the given segmentation extractors.""" images_in_extractor1 = segmentation_extractor1.get_images_dict() images_in_extractor2 = segmentation_extractor2.get_images_dict() @@ -243,11 +286,7 @@ def check_segmentations_images( def check_segmentation_return_types(seg: SegmentationExtractor): - """ - Parameters - ---------- - seg:SegmentationExtractor - """ + """Check that the return types of the segmentation extractor are correct.""" assert isinstance(seg.get_num_rois(), int) assert isinstance(seg.get_num_frames(), int) assert isinstance(seg.get_num_channels(), int) @@ -317,6 +356,7 @@ def check_segmentation_return_types(seg: SegmentationExtractor): def check_imaging_equal( imaging_extractor1: ImagingExtractor, imaging_extractor2: ImagingExtractor, exclude_channel_comparison: bool = False ): + """Check that two imaging extractors have equal fields.""" # assert equality: assert imaging_extractor1.get_num_frames() == imaging_extractor2.get_num_frames() assert imaging_extractor1.get_num_channels() == imaging_extractor2.get_num_channels() @@ -337,15 +377,10 @@ def check_imaging_equal( def assert_get_frames_return_shape(imaging_extractor: ImagingExtractor): - """Utiliy to check whether an ImagingExtractor get_frames function behaves as expected. We aim for the function to - behave as numpy slicing and indexing as much as possible + """Check whether an ImagingExtractor get_frames function behaves as expected. - Parameters - ---------- - imaging_extractor : ImagingExtractor - An image extractor + We aim for the function to behave as numpy slicing and indexing as much as possible. """ - image_size = imaging_extractor.get_image_size() frame_idxs = 0 @@ -369,11 +404,7 @@ def assert_get_frames_return_shape(imaging_extractor: ImagingExtractor): def check_imaging_return_types(img_ex: ImagingExtractor): - """ - Parameters - ---------- - img_ex:ImagingExtractor - """ + """Check that the return types of the imaging extractor are correct.""" assert isinstance(img_ex.get_num_frames(), inttype) assert isinstance(img_ex.get_num_channels(), inttype) assert isinstance(img_ex.get_sampling_frequency(), floattype) From 957c53c33c17eeb3075d4bb1a99935a125989368 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 5 Sep 2023 14:48:31 -0700 Subject: [PATCH 37/39] added docstrings to testing --- src/roiextractors/testing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/roiextractors/testing.py b/src/roiextractors/testing.py index 8d30a9cb..d1720898 100644 --- a/src/roiextractors/testing.py +++ b/src/roiextractors/testing.py @@ -1,3 +1,4 @@ +"""Testing utilities for the roiextractors package.""" from collections.abc import Iterable from typing import Tuple, Optional From 47eaa3d2111aee3375ee1e7bfcb60f17819f0fbd Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Wed, 6 Sep 2023 11:54:26 -0700 Subject: [PATCH 38/39] TODO for mode parameter in toy_example --- src/roiextractors/example_datasets/toy_example.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/roiextractors/example_datasets/toy_example.py b/src/roiextractors/example_datasets/toy_example.py index 968c77cc..e8a9a58d 100644 --- a/src/roiextractors/example_datasets/toy_example.py +++ b/src/roiextractors/example_datasets/toy_example.py @@ -18,7 +18,9 @@ def _gaussian(x, mu, sigma): return 1 / np.sqrt(2 * np.pi * sigma) * np.exp(-((x - mu) ** 2) / sigma) -def _generate_rois(num_units=10, size_x=100, size_y=100, roi_size=4, min_dist=5, mode="uniform"): +def _generate_rois( + num_units=10, size_x=100, size_y=100, roi_size=4, min_dist=5, mode="uniform" +): # TODO: mode --> literal type """Generate ROIs with given parameters. Parameters From 2769b8e105ac841c55d41e5f8b1051bd666f0693 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Thu, 7 Sep 2023 12:12:31 -0700 Subject: [PATCH 39/39] removed docstring test to simplify PR --- tests/test_docstrings.py | 61 ---------------------------------------- 1 file changed, 61 deletions(-) delete mode 100644 tests/test_docstrings.py diff --git a/tests/test_docstrings.py b/tests/test_docstrings.py deleted file mode 100644 index 26588dc3..00000000 --- a/tests/test_docstrings.py +++ /dev/null @@ -1,61 +0,0 @@ -import inspect -import os -import importlib -import roiextractors -import pytest - - -def traverse_class(cls, objs): - """Traverse a class and its methods and append them to objs.""" - predicate = lambda x: inspect.isfunction(x) or inspect.ismethod(x) - for name, obj in inspect.getmembers(cls, predicate=predicate): - objs.append(obj) - - -def traverse_module(module, objs): - """Traverse all classes and functions in a module and append them to objs.""" - objs.append(module) - predicate = lambda x: inspect.isclass(x) or inspect.isfunction(x) or inspect.ismethod(x) - for name, obj in inspect.getmembers(module, predicate=predicate): - parent_package = obj.__module__.split(".")[0] - if parent_package != "roiextractors": # avoid traversing external dependencies - continue - objs.append(obj) - if inspect.isclass(obj): - traverse_class(obj, objs) - - -def traverse_package(package, objs): - """Traverse all modules and subpackages in a package to append all members to objs.""" - for child in os.listdir(package.__path__[0]): - if child.startswith(".") or child == "__pycache__": - continue - elif child.endswith(".py"): - module_name = child[:-3] - module = importlib.import_module(f"{package.__name__}.{module_name}") - traverse_module(module, objs) - else: # subpackage - subpackage = importlib.import_module(f"{package.__name__}.{child}") - traverse_package(subpackage, objs) - - -objs = [] -traverse_package(roiextractors, objs) - - -@pytest.mark.parametrize("obj", objs) -def test_has_docstring(obj): - """Check if an object has a docstring.""" - doc = inspect.getdoc(obj) - if inspect.ismodule(obj): - msg = f"{obj.__name__} has no docstring." - else: - msg = f"{obj.__module__}.{obj.__qualname__} has no docstring." - if "__create_fn__" in msg: - return # skip dataclass functions created by __create_fn__ - assert doc is not None, msg - - -if __name__ == "__main__": - for obj in objs: - test_has_docstring(obj)