From 22cfb7ce75b5e62cc73853beca288cc70f35da2f Mon Sep 17 00:00:00 2001 From: Maxim Lippeveld Date: Wed, 2 Nov 2022 14:17:56 +0100 Subject: [PATCH] Documentation update --- README.md | 13 ++++-- docs/Makefile | 15 ++++--- docs/source/conf.py | 2 - docs/source/generated/scip.export.rst | 13 +++--- docs/source/generated/scip.features.rst | 21 ++++++---- docs/source/generated/scip.filter.rst | 13 +++--- .../scip.illumination_correction.rst | 13 +++--- docs/source/generated/scip.loading.rst | 13 +++--- .../source/generated/scip.masking.filters.rst | 34 +++++++++++++++ docs/source/generated/scip.masking.rst | 41 ++++++++++++------- docs/source/generated/scip.normalization.rst | 13 +++--- docs/source/generated/scip.projection.rst | 13 +++--- docs/source/generated/scip.rst | 13 +++--- docs/source/generated/scip.segmentation.rst | 13 +++--- docs/source/generated/scip.utils.rst | 13 +++--- docs/source/index.rst | 10 +---- docs/source/input.rst | 2 +- docs/source/installation.rst | 23 ++--------- docs/source/usage.rst | 25 +++++++++++ .../illumination_correction/jones_2006.py | 5 ++- src/scip/masking/filters/normaltest.py | 2 + src/scip/masking/filters/std.py | 2 + src/scip/masking/filters/value_range.py | 2 + src/scip/projection/op.py | 7 ++++ src/scip/segmentation/cellpose.py | 20 ++++++++- 25 files changed, 205 insertions(+), 136 deletions(-) create mode 100644 docs/source/generated/scip.masking.filters.rst diff --git a/README.md b/README.md index 3d8e632..5a1b414 100644 --- a/README.md +++ b/README.md @@ -8,19 +8,26 @@ Scalable Cytometry Image Processing (SCIP) is an open-source tool that implement an image processing pipeline on top of Dask, a distributed computing framework written in Python. SCIP performs normalization, image segmentation and masking, and feature extraction. -Check the docs for installation and usage instructions. +Check the [docs](https://readthedocs.org/projects/scalable-cytometry-image-processing) for +installation and usage instructions, and API documentation. ## Development ### Generating documentation + +For publishing online: ``` cd docs -rm -r source/generated/ -sphinx-apidoc -f -T -o source/generated/ ../src/scip make clean make html ``` +For development with live reloading: +``` +cd docs +make livehtml +``` + ### Generate release changelog ``` git log v{previous version tag}..HEAD --oneline | xclip -sel clip diff --git a/docs/Makefile b/docs/Makefile index 5249f95..32ac563 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -5,6 +5,7 @@ # from the environment for the first two. SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build +SPHINXAPIDOC = sphinx-apidoc SOURCEDIR = source BUILDDIR = build @@ -14,10 +15,14 @@ help: .PHONY: help Makefile -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile +html: api @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -livehtml: - sphinx-autobuild "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) +api: + @$(SPHINXAPIDOC) -f -T -M -o "$(SOURCEDIR)/generated" ../src/scip + +livehtml: api + sphinx-autobuild --watch ../src "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +clean: + rm -r "$(SOURCEDIR)/generated" $(BUILDDIR) diff --git a/docs/source/conf.py b/docs/source/conf.py index be8ce0a..d7ddcf9 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -28,9 +28,7 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', 'sphinx.ext.napoleon', - 'sphinx.ext.autosummary', 'sphinx_copybutton' ] diff --git a/docs/source/generated/scip.export.rst b/docs/source/generated/scip.export.rst index 53f7ecc..5aa8c82 100644 --- a/docs/source/generated/scip.export.rst +++ b/docs/source/generated/scip.export.rst @@ -1,6 +1,11 @@ scip.export package =================== +.. automodule:: scip.export + :members: + :undoc-members: + :show-inheritance: + Submodules ---------- @@ -19,11 +24,3 @@ scip.export.parquet module :members: :undoc-members: :show-inheritance: - -Module contents ---------------- - -.. automodule:: scip.export - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/generated/scip.features.rst b/docs/source/generated/scip.features.rst index 384641e..f389e1c 100644 --- a/docs/source/generated/scip.features.rst +++ b/docs/source/generated/scip.features.rst @@ -1,6 +1,11 @@ scip.features package ===================== +.. automodule:: scip.features + :members: + :undoc-members: + :show-inheritance: + Submodules ---------- @@ -20,6 +25,14 @@ scip.features.intensity module :undoc-members: :show-inheritance: +scip.features.raw\_intensity module +----------------------------------- + +.. automodule:: scip.features.raw_intensity + :members: + :undoc-members: + :show-inheritance: + scip.features.shape module -------------------------- @@ -35,11 +48,3 @@ scip.features.texture module :members: :undoc-members: :show-inheritance: - -Module contents ---------------- - -.. automodule:: scip.features - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/generated/scip.filter.rst b/docs/source/generated/scip.filter.rst index decbc51..23c111e 100644 --- a/docs/source/generated/scip.filter.rst +++ b/docs/source/generated/scip.filter.rst @@ -1,6 +1,11 @@ scip.filter package =================== +.. automodule:: scip.filter + :members: + :undoc-members: + :show-inheritance: + Submodules ---------- @@ -11,11 +16,3 @@ scip.filter.threshold module :members: :undoc-members: :show-inheritance: - -Module contents ---------------- - -.. automodule:: scip.filter - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/generated/scip.illumination_correction.rst b/docs/source/generated/scip.illumination_correction.rst index 963771a..742ee10 100644 --- a/docs/source/generated/scip.illumination_correction.rst +++ b/docs/source/generated/scip.illumination_correction.rst @@ -1,6 +1,11 @@ scip.illumination\_correction package ===================================== +.. automodule:: scip.illumination_correction + :members: + :undoc-members: + :show-inheritance: + Submodules ---------- @@ -11,11 +16,3 @@ scip.illumination\_correction.jones\_2006 module :members: :undoc-members: :show-inheritance: - -Module contents ---------------- - -.. automodule:: scip.illumination_correction - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/generated/scip.loading.rst b/docs/source/generated/scip.loading.rst index 9007289..7ba2d98 100644 --- a/docs/source/generated/scip.loading.rst +++ b/docs/source/generated/scip.loading.rst @@ -1,6 +1,11 @@ scip.loading package ==================== +.. automodule:: scip.loading + :members: + :undoc-members: + :show-inheritance: + Submodules ---------- @@ -43,11 +48,3 @@ scip.loading.zarr module :members: :undoc-members: :show-inheritance: - -Module contents ---------------- - -.. automodule:: scip.loading - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/generated/scip.masking.filters.rst b/docs/source/generated/scip.masking.filters.rst new file mode 100644 index 0000000..c1cfec2 --- /dev/null +++ b/docs/source/generated/scip.masking.filters.rst @@ -0,0 +1,34 @@ +scip.masking.filters package +============================ + +.. automodule:: scip.masking.filters + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +scip.masking.filters.normaltest module +-------------------------------------- + +.. automodule:: scip.masking.filters.normaltest + :members: + :undoc-members: + :show-inheritance: + +scip.masking.filters.std module +------------------------------- + +.. automodule:: scip.masking.filters.std + :members: + :undoc-members: + :show-inheritance: + +scip.masking.filters.value\_range module +---------------------------------------- + +.. automodule:: scip.masking.filters.value_range + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/generated/scip.masking.rst b/docs/source/generated/scip.masking.rst index 633e3af..a3950e6 100644 --- a/docs/source/generated/scip.masking.rst +++ b/docs/source/generated/scip.masking.rst @@ -1,6 +1,19 @@ scip.masking package ==================== +.. automodule:: scip.masking + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + scip.masking.filters + Submodules ---------- @@ -12,42 +25,42 @@ scip.masking.circle module :undoc-members: :show-inheritance: -scip.masking.sobel module -------------------------- +scip.masking.li module +---------------------- -.. automodule:: scip.masking.sobel +.. automodule:: scip.masking.li :members: :undoc-members: :show-inheritance: -scip.masking.spot module +scip.masking.otsu module ------------------------ -.. automodule:: scip.masking.spot +.. automodule:: scip.masking.otsu :members: :undoc-members: :show-inheritance: -scip.masking.threshold module ------------------------------ +scip.masking.sobel module +------------------------- -.. automodule:: scip.masking.threshold +.. automodule:: scip.masking.sobel :members: :undoc-members: :show-inheritance: -scip.masking.watershed module ------------------------------ +scip.masking.spot module +------------------------ -.. automodule:: scip.masking.watershed +.. automodule:: scip.masking.spot :members: :undoc-members: :show-inheritance: -Module contents ---------------- +scip.masking.watershed module +----------------------------- -.. automodule:: scip.masking +.. automodule:: scip.masking.watershed :members: :undoc-members: :show-inheritance: diff --git a/docs/source/generated/scip.normalization.rst b/docs/source/generated/scip.normalization.rst index e452aa0..587daf4 100644 --- a/docs/source/generated/scip.normalization.rst +++ b/docs/source/generated/scip.normalization.rst @@ -1,6 +1,11 @@ scip.normalization package ========================== +.. automodule:: scip.normalization + :members: + :undoc-members: + :show-inheritance: + Submodules ---------- @@ -11,11 +16,3 @@ scip.normalization.quantile\_normalization module :members: :undoc-members: :show-inheritance: - -Module contents ---------------- - -.. automodule:: scip.normalization - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/generated/scip.projection.rst b/docs/source/generated/scip.projection.rst index 22c0b09..1758d7d 100644 --- a/docs/source/generated/scip.projection.rst +++ b/docs/source/generated/scip.projection.rst @@ -1,6 +1,11 @@ scip.projection package ======================= +.. automodule:: scip.projection + :members: + :undoc-members: + :show-inheritance: + Submodules ---------- @@ -11,11 +16,3 @@ scip.projection.op module :members: :undoc-members: :show-inheritance: - -Module contents ---------------- - -.. automodule:: scip.projection - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/generated/scip.rst b/docs/source/generated/scip.rst index ce1226a..f26e9a4 100644 --- a/docs/source/generated/scip.rst +++ b/docs/source/generated/scip.rst @@ -1,6 +1,11 @@ scip package ============ +.. automodule:: scip + :members: + :undoc-members: + :show-inheritance: + Subpackages ----------- @@ -28,11 +33,3 @@ scip.main module :members: :undoc-members: :show-inheritance: - -Module contents ---------------- - -.. automodule:: scip - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/generated/scip.segmentation.rst b/docs/source/generated/scip.segmentation.rst index a5717c7..91c04d9 100644 --- a/docs/source/generated/scip.segmentation.rst +++ b/docs/source/generated/scip.segmentation.rst @@ -1,6 +1,11 @@ scip.segmentation package ========================= +.. automodule:: scip.segmentation + :members: + :undoc-members: + :show-inheritance: + Submodules ---------- @@ -19,11 +24,3 @@ scip.segmentation.watershed\_dapi module :members: :undoc-members: :show-inheritance: - -Module contents ---------------- - -.. automodule:: scip.segmentation - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/generated/scip.utils.rst b/docs/source/generated/scip.utils.rst index c0f6b68..1785870 100644 --- a/docs/source/generated/scip.utils.rst +++ b/docs/source/generated/scip.utils.rst @@ -1,6 +1,11 @@ scip.utils package ================== +.. automodule:: scip.utils + :members: + :undoc-members: + :show-inheritance: + Submodules ---------- @@ -11,11 +16,3 @@ scip.utils.util module :members: :undoc-members: :show-inheritance: - -Module contents ---------------- - -.. automodule:: scip.utils - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/index.rst b/docs/source/index.rst index e8528a6..4736860 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -9,13 +9,7 @@ Table of contents :maxdepth: 1 installation - usage input + usage generated/scip - -Indices and tables ------------------- - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` + Github diff --git a/docs/source/input.rst b/docs/source/input.rst index 1b31737..ff019a1 100644 --- a/docs/source/input.rst +++ b/docs/source/input.rst @@ -16,7 +16,7 @@ images can be grouped into a couple of zarr files, which are much easier to hand The images are stored in the zarr file using the VLenArray object codec, which supports arrays of varying length. This is described in `the zarr documentation `_. This way the images don't have to be cropped or padded to account -for their varying X and Y dimensions. The images do have to be flattened as the coded only +for their varying X and Y dimensions. The images do have to be flattened as the codec only supports 1D arrays. SCIP reshapes the images upon loading using the shape attribute described below. The zarr file should have two attributes: diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 1736b16..88840c6 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -2,7 +2,7 @@ Installation ============ -SCIP can be installed from the PyPi and conda-forge repositories, +SCIP can be installed from the PyPi repository, or from source. We recommend installing SCIP in a conda environment @@ -22,27 +22,10 @@ an MPI implementation must be available. We recommend installing mpich. mamba install -c conda-forge mpich-mpicc -conda-forge ------------ - -You can install SCIP in a new environment, - -.. code-block:: - - mamba env create -n scip -c conda-forge scip - -or in an existing environment: -.. code-block:: - - mamba activate env - mamba install -c conda-forge scip - -This will install the full SCIP package. - PyPi ---- -SCIP can also be installed from the PyPi repository. The base +SCIP can be installed from the PyPi repository. The base installation can be installed as follows: .. code-block:: @@ -72,5 +55,5 @@ To install SCIP from source: 2. Optionally extract the release. 3. Enter the repository directory. 4. Run ``pip install .`` if you only need default functionality. -5. (Optional) Run ``pip install .[extra]`` for extra functionality (refer to the list above +5. (Optional) Run ``pip install .[extra]`` for extra functionality (refer to the list above for available extras) diff --git a/docs/source/usage.rst b/docs/source/usage.rst index da434bb..f9fccc8 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -4,6 +4,9 @@ Usage SCIP can be used as a command line interface (CLI) or its modules can be imported in a custom script. See the package API documentation for the latter option. +Configuration +============= + The CLI runs a workflow that loads images and performs projection, illumination correction, segmentation, masking, and feature extraction. The CLI can be configured via options passed on the command line (for runtime configuration) and via a YAML config file (for pipeline configuration). For an overview of the command line options, run ``scip --help``. The YAML config file has the following specification: @@ -82,3 +85,25 @@ Note that all top-level keys have to be present. You can disable a step by only .. code-block:: yaml mask: + +Command-line +============ + +This package exposes one command-line client (CLI): :code:`scip` + +Call + +.. code-block:: bash + + scip --help + +to get an overview of all available options. + +The structure of the CLI is as follows: + +.. code-block:: bash + + scip [OPTIONS] OUTPUT CONFIG [PATHS]... + +All runtime options, such as the number of workers are passed in the :code:`OPTIONS` section. All +pipeline settings are passed using the configuration file passed through :code:`CONFIG`. \ No newline at end of file diff --git a/src/scip/illumination_correction/jones_2006.py b/src/scip/illumination_correction/jones_2006.py index f653622..5bf7fb2 100644 --- a/src/scip/illumination_correction/jones_2006.py +++ b/src/scip/illumination_correction/jones_2006.py @@ -66,10 +66,13 @@ def correct( precomputed: Path = None ) -> dask.bag.Bag: """ - Distributed implementation of Jones et al. (2006) illumination correction. All images + Distributed implementation of retrospective illumination correction [1]. All images are averaged per batch, after which the image is filtered using a median filter. If requested, the image is downscaled prior to median filtering to reduce memory consumption. + [1] Singh, S., Bray, M. A., Jones, T. R., & Carpenter, A. E. (2014). Pipeline for illumination + correction of images for high‐throughput microscopy. Journal of microscopy, 256(3), 231-236. + Args: images: Collection containing images to be corrected. Each item in the collection a pixels and 'key' key. diff --git a/src/scip/masking/filters/normaltest.py b/src/scip/masking/filters/normaltest.py index a7e3792..ba45fef 100644 --- a/src/scip/masking/filters/normaltest.py +++ b/src/scip/masking/filters/normaltest.py @@ -5,4 +5,6 @@ def filter( x: numpy.ndarray ) -> bool: + """Performs normality test on pixel values at .05 significance level. + Returns True if pixels are not normally distributed.""" return normaltest(x.ravel()).pvalue < 0.05 diff --git a/src/scip/masking/filters/std.py b/src/scip/masking/filters/std.py index d0fc8af..f171257 100644 --- a/src/scip/masking/filters/std.py +++ b/src/scip/masking/filters/std.py @@ -5,4 +5,6 @@ def filter( x: numpy.ndarray, threshold: float ) -> bool: + """Checks standard deviation of pixel values against threshold. + Returns True if standard deviation is above threshold.""" return x.std() > threshold diff --git a/src/scip/masking/filters/value_range.py b/src/scip/masking/filters/value_range.py index d490130..738ae33 100644 --- a/src/scip/masking/filters/value_range.py +++ b/src/scip/masking/filters/value_range.py @@ -5,4 +5,6 @@ def filter( x: numpy.ndarray, threshold: float ) -> bool: + """Checks range width of pixel values against threshold. + Returns True if width is above threshold.""" return (x.max() - x.min()) > threshold diff --git a/src/scip/projection/op.py b/src/scip/projection/op.py index e6966f9..bf12ef9 100644 --- a/src/scip/projection/op.py +++ b/src/scip/projection/op.py @@ -32,6 +32,13 @@ def project_block( event: Mapping[str, Any], op: str ) -> Mapping[str, Any]: + """Performs Z-stack projection of multi focal pixel data. + + Args: + op: Projection operation to be performed (one of max, mean). + Returns: + Events with projected pixels. + """ newevent = copy_without(event, without=["pixels"]) newevent["pixels"] = _OPS[op](event["pixels"]) diff --git a/src/scip/segmentation/cellpose.py b/src/scip/segmentation/cellpose.py index 59b48b0..7f00db5 100644 --- a/src/scip/segmentation/cellpose.py +++ b/src/scip/segmentation/cellpose.py @@ -33,7 +33,23 @@ def segment_block( cell_diameter: Optional[int] = None, flow_threshold: Optional[float] = 0.4, **kwargs -) -> List[dict]: +) -> List[Mapping[str, Any]]: + """Performs CellPose[1] segmentation. + + [1] Stringer, C., Wang, T., Michaelos, M., & Pachitariu, M. (2021). Cellpose: a generalist + algorithm for cellular segmentation. Nature methods, 18(1), 100-106. + + Args: + channel_indices: Indices of channels to be segmented. + parent_channel_index: Index of parent channel. Objects detected in other + channels which overlap with an object detected in this channel will be assigned to it. + dapi_channel_index: Index of DAPI channel. + gpu_accelerated: Whether segmentation should be run on GPU. + cell_diameter: See CellPose documentation. + flow_threshold: See CellPose documentation. + Returns: + Events with mask obtained using CellPose. + """ if len(events) == 0: return events @@ -88,7 +104,7 @@ def segment_block( for channel_index, child in children: - # assign over-segmented children to parent objects + # assign children to parent objects mask = numpy.zeros_like(parents[e_i]) for i in numpy.unique(parents[e_i])[1:]: idx, counts = numpy.unique(