From ad56927ee4923c447c4bae53119de6c6607c5758 Mon Sep 17 00:00:00 2001 From: Roy Hoitink Date: Fri, 16 Feb 2024 17:11:39 +0100 Subject: [PATCH 1/6] Implemented new way of noise simulation --- simulatedmicroscopy/image.py | 80 ++++++++++++++++++++++++++++++++---- simulatedmicroscopy/psf.py | 4 +- tests/test_image.py | 22 ++++++++-- 3 files changed, 91 insertions(+), 15 deletions(-) diff --git a/simulatedmicroscopy/image.py b/simulatedmicroscopy/image.py index e7a9c66..2732195 100644 --- a/simulatedmicroscopy/image.py +++ b/simulatedmicroscopy/image.py @@ -22,8 +22,11 @@ class Image: is_downsampled = False """Whether the image has undergone downsampling""" - has_noise = False - """Whether the image has undergone noise addition""" + has_shot_noise = False + """Whether the image has undergone shot noise addition""" + + has_read_noise = False + """Whether the image has undergone read noise addition""" def __init__( self, @@ -382,7 +385,7 @@ def convolve(self, other: type[Image]) -> type[Image]: return self def noisify(self, lam: float = 1.0) -> type[Image]: - """Add Poisson noise to the image + """Add Poisson noise to the image [DEPRECATED] Parameters ---------- @@ -394,14 +397,73 @@ def noisify(self, lam: float = 1.0) -> type[Image]: type[Image] The image with noise added """ - if self.has_noise: - warnings.warn("Image has already undergone noisification once") + raise DeprecationWarning( + "This method is deprecated, please use either add_shot_noise or add_read_noise" + ) + + def add_shot_noise(self, SNR: float = 30.0) -> type[Image]: + """Add shot noise to the image, realised by sampling the intensities of the image from a Poisson distribution. + At each pixel, the current value (after rescaling) dictates the mean value of the Poisson distribution, yielding + signal-dependent noise. + + Parameters + ---------- + SNR : float, optional + The signal-to-noise ratio of the resulting image, by default 30.0. + + Returns + ------- + type[Image] + The image with shot noise + """ + if self.has_shot_noise: + warnings.warn("This image already has shot noise, proceeding anyway") + + scaling_factor = ( + SNR**2 / self.image.mean() + ) # scaling factor to get a proper Poisson sampling + + self.image = ( + np.random.poisson(scaling_factor * self.image.mean()) / scaling_factor + ) + # division by `scaling_factor` to revert pixel values back to their (roughly) original range + + self.has_shot_noise = True # set shot noise flag + + return self + + def add_read_noise(self, SNR: float = 50.0, background: float = 0.0) -> type[Image]: + """Add read noise to the image, realised by adding noise sampled from a normal distribution. The standard deviation + of the distribution is constant across the entire image and determined by the SNR parameter, it is therefore + signal-independent. + + Parameters + ---------- + SNR : float, optional + The signal-to-noise ratio of the resulting image, by default 50.0 + background : float, optional + Optional offset of the background, this will be the mean of the normal distribution, by default 0.0 + + Returns + ------- + type[Image] + The image with read noise + """ + if self.has_read_noise: + warnings.warn("This image already has read noise, proceeding anyway") + + peak = np.percentile( + self.image, 99.9 + ) # get peak intensity at the 99.9 percentile + + # As SNR = peak / sigma; this yields that sigma = peak/SNR + sigma = peak / SNR - rng = np.random.default_rng() - self.image = self.image * rng.poisson(lam, size=self.image.shape) + self.image += np.random.normal( + loc=background, scale=sigma + ) # centred around value of `background` - self.has_noise = True - self.metadata["has_noise"] = True + self.has_read_noise = True # set read noise flag return self diff --git a/simulatedmicroscopy/psf.py b/simulatedmicroscopy/psf.py index 32870ca..679df7c 100644 --- a/simulatedmicroscopy/psf.py +++ b/simulatedmicroscopy/psf.py @@ -54,9 +54,7 @@ def __init__( mu = [0.0] * 3 # generate Gaussian with given mu and sigmas - gauss_psf = scipy.stats.multivariate_normal.pdf( - zyx, mu, np.diag(sigmas_nm**2) - ) + gauss_psf = scipy.stats.multivariate_normal.pdf(zyx, mu, np.diag(sigmas_nm**2)) # reshape to 3D image gauss_psf = gauss_psf.reshape(z.shape) diff --git a/tests/test_image.py b/tests/test_image.py index 4825600..40d4c78 100644 --- a/tests/test_image.py +++ b/tests/test_image.py @@ -154,18 +154,34 @@ def test_convolution_wrongpixelsize(): im.convolve(psf) -def test_noise(): +def test_noise_deprecated(): + with pytest.raises(DeprecationWarning): + im = create_demo_image() + im.noisify() + +def test_shot_noise(): im = create_demo_image() # check if returns correct image - assert im.noisify() != create_demo_image() + assert im.add_shot_noise(SNR = 10.0) != create_demo_image() # check if original image was also changed assert im != create_demo_image() # should be set to True - assert im.has_noise + assert im.has_shot_noise + +def test_read_noise(): + im = create_demo_image() + + # check if returns correct image + assert im.add_read_noise(SNR = 10.0, background = 1e-3) != create_demo_image() + # check if original image was also changed + assert im != create_demo_image() + + # should be set to True + assert im.has_read_noise def test_point_image(tmp_path): from simulatedmicroscopy import Coordinates From ab036a4db0674e4145a11106926b0f1b68a20ea7 Mon Sep 17 00:00:00 2001 From: Roy Hoitink Date: Mon, 19 Feb 2024 10:30:39 +0100 Subject: [PATCH 2/6] Update docs on noise and prevent negative values in Poisson distribution --- docs/data-generation-example.md | 7 ++++--- simulatedmicroscopy/image.py | 6 +++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/data-generation-example.md b/docs/data-generation-example.md index 29867c9..b0326ad 100644 --- a/docs/data-generation-example.md +++ b/docs/data-generation-example.md @@ -86,12 +86,13 @@ You convolved image is now saved in the `image` variable. ## 4. Optional: adding noise and downsampling the image -Adding Poisson noise and downsampling the image to a lower pixel size is included as well and can be done as follows: +Adding noise and downsampling the image to a lower pixel size is included as well and can be done as follows: ```python -# add Poisson noise with λ = 50 # downsample by a factor 2 in z and factor 3 in xy -image.noisify(50).downsample([2, 3, 3]) +# add Poisson noise (shot noise) with a signal-to-noise ratio of 30 +# add Gaussian noise (read noise, additive) with a mean value of 1e-5 and a signal-to-noise ratio of 50 +image.downsample([2, 3, 3]).add_shot_noise(SNR = 30.0).add_read_noise(SNR = 50.0, background = 1e-5) ``` The downsampling by a factor two along a dimension, means that the number of pixels along that dimension is divided by two, which leads to an increase of the pixel size by a factor or two. diff --git a/simulatedmicroscopy/image.py b/simulatedmicroscopy/image.py index 2732195..2bf4c96 100644 --- a/simulatedmicroscopy/image.py +++ b/simulatedmicroscopy/image.py @@ -419,6 +419,10 @@ def add_shot_noise(self, SNR: float = 30.0) -> type[Image]: if self.has_shot_noise: warnings.warn("This image already has shot noise, proceeding anyway") + # Poisson cannot deal with negative values, if these are present, shift whole image + if np.any(self.image < 0.0): + self.image += np.abs(self.image.min()) + scaling_factor = ( SNR**2 / self.image.mean() ) # scaling factor to get a proper Poisson sampling @@ -426,7 +430,7 @@ def add_shot_noise(self, SNR: float = 30.0) -> type[Image]: self.image = ( np.random.poisson(scaling_factor * self.image.mean()) / scaling_factor ) - # division by `scaling_factor` to revert pixel values back to their (roughly) original range + # division by `scaling_factor` at the end to revert pixel values back to their (roughly) original range self.has_shot_noise = True # set shot noise flag From 5e7fa38547c41450faa750dd780f284d8aa763f0 Mon Sep 17 00:00:00 2001 From: Roy Hoitink Date: Mon, 19 Feb 2024 10:35:50 +0100 Subject: [PATCH 3/6] Add testing for Python 3.12 --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 1d782a5..43f6f65 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 From cf58cc0bfcd0ff5cd1e4b9add2ad66568bb7161a Mon Sep 17 00:00:00 2001 From: Roy Hoitink Date: Mon, 19 Feb 2024 11:14:00 +0100 Subject: [PATCH 4/6] Switched to pyproject.toml based setup --- .pre-commit-config.yaml | 16 +++++ pyproject.toml | 57 +++++++++++++++++ setup.cfg | 63 ++++++++++++++++++- setup.py | 34 ---------- .../simulatedmicroscopy}/__init__.py | 8 ++- src/simulatedmicroscopy/_version.py | 16 +++++ .../simulatedmicroscopy}/image.py | 0 .../simulatedmicroscopy}/input.py | 0 .../simulatedmicroscopy}/particle.py | 0 .../simulatedmicroscopy}/psf.py | 0 10 files changed, 155 insertions(+), 39 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 pyproject.toml delete mode 100644 setup.py rename {simulatedmicroscopy => src/simulatedmicroscopy}/__init__.py (78%) create mode 100644 src/simulatedmicroscopy/_version.py rename {simulatedmicroscopy => src/simulatedmicroscopy}/image.py (100%) rename {simulatedmicroscopy => src/simulatedmicroscopy}/input.py (100%) rename {simulatedmicroscopy => src/simulatedmicroscopy}/particle.py (100%) rename {simulatedmicroscopy => src/simulatedmicroscopy}/psf.py (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..0430273 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,16 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-docstring-first + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-yaml # checks for correct yaml syntax for github actions ex. + - repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.2.2 + hooks: + - id: ruff + - repo: https://github.com/psf/black + rev: 23.3.0 + hooks: + - id: black \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..77666c3 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,57 @@ +[build-system] +requires = ["setuptools>=42.0.0", "wheel", "setuptools_scm"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +write_to = "src/simulatedmicroscopy/_version.py" + +[tool.black] +line-length = 79 +target-version = ['py38', 'py39', 'py310', 'py311', 'py312'] + +[tool.ruff] +line-length = 79 +select = [ + "E", "F", "W", #flake8 + "UP", # pyupgrade + "I", # isort + "BLE", # flake8-blind-exception + "B", # flake8-bugbear + "A", # flake8-builtins + "C4", # flake8-comprehensions + "ISC", # flake8-implicit-str-concat + "G", # flake8-logging-format + "PIE", # flake8-pie + "SIM", # flake8-simplify +] + +ignore = [ + "E501", + "E203", + "W503", +] + +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".mypy_cache", + ".pants.d", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "venv", + "*vendored*", + "*_vendor*", +] + +target-version = "py38" +fix = true diff --git a/setup.cfg b/setup.cfg index b844578..da8b4a3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,61 @@ -[flake8] -ignore=E501,E203,W503 \ No newline at end of file +[metadata] +name = simulatedmicroscopy + +description = Python package to create synthetic (fluorescence) microscopy images of (nano)particles and convolution with a point spread function +long_description = file: README.md +long_description_content_type = text/markdown +url = https://github.com/rhoitink/simulatedmicroscopy +author = Roy Hoitink +author_email = L.D.Hoitink@uu.nl +license = MIT +license_files = LICENSE +classifiers = + Development Status :: 4 - Beta + Intended Audience :: Science/Research + License :: OSI Approved :: MIT License + Operating System :: OS Independent + Programming Language :: Python + Programming Language :: Python :: 3 + Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 + Topic :: Scientific/Engineering :: Image Processing +project_urls = + Bug Tracker = https://github.com/rhoitink/simulatedmicroscopy/issues + Documentation = https://github.com/rhoitink/simulatedmicroscopy#README.md + Source Code = https://github.com/rhoitink/simulatedmicroscopy + User Support = https://github.com/rhoitink/simulatedmicroscopy/issues + +[options] +packages = find: +install_requires = + numpy + matplotlib + h5py + scipy + scikit-image>=0.20.0 + +python_requires = >=3.8 +include_package_data = True +package_dir = + =src +setup_requires = setuptools_scm + +[options.packages.find] +where = src + +[options.extras_require] +dev = + black + flake8 + pytest + pytest-cov +docs = + mkdocs + mkdocs-material + mkdocstrings[python] + mkdocs-gen-files + mkdocs-literate-nav \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index de801b3..0000000 --- a/setup.py +++ /dev/null @@ -1,34 +0,0 @@ -from setuptools import setup, find_packages - -setup( - name="simulatedmicroscopy", - version="1.4.0", - author="Roy Hoitink", - author_email="L.D.Hoitink@uu.nl", - long_description=open("README.md").read(), - packages=find_packages(include=["simulatedmicroscopy", "simulatedmicroscopy.*"]), - python_requires=">=3.8", - install_requires=[ - "numpy", - "matplotlib", - "h5py", - "scipy", - "scikit-image>=0.20.0", - ], - extras_require={ - "dev": [ - "black", - "flake8", - "pytest", - "pytest-cov", - "bump2version", - ], - "docs": [ - "mkdocs", - "mkdocs-material", - "mkdocstrings[python]", - "mkdocs-gen-files", - "mkdocs-literate-nav", - ] - }, -) diff --git a/simulatedmicroscopy/__init__.py b/src/simulatedmicroscopy/__init__.py similarity index 78% rename from simulatedmicroscopy/__init__.py rename to src/simulatedmicroscopy/__init__.py index 9484a7c..c243d25 100644 --- a/simulatedmicroscopy/__init__.py +++ b/src/simulatedmicroscopy/__init__.py @@ -2,6 +2,10 @@ from .image import Image, HuygensImage from .psf import HuygensPSF, GaussianPSF from .particle import Sphere, Shell, PointParticle, Spherocylinder, Cube +try: + from ._version import version as __version__ +except ImportError: + __version__ = "unknown" __all__ = [ "Coordinates", @@ -14,6 +18,4 @@ "Shell", "Spherocylinder", "Cube", -] - -__version__ = "1.4.0" +] \ No newline at end of file diff --git a/src/simulatedmicroscopy/_version.py b/src/simulatedmicroscopy/_version.py new file mode 100644 index 0000000..081efef --- /dev/null +++ b/src/simulatedmicroscopy/_version.py @@ -0,0 +1,16 @@ +# file generated by setuptools_scm +# don't change, don't track in version control +TYPE_CHECKING = False +if TYPE_CHECKING: + from typing import Tuple, Union + VERSION_TUPLE = Tuple[Union[int, str], ...] +else: + VERSION_TUPLE = object + +version: str +__version__: str +__version_tuple__: VERSION_TUPLE +version_tuple: VERSION_TUPLE + +__version__ = version = '1.4.1.dev7+g5e7fa38.d20240219' +__version_tuple__ = version_tuple = (1, 4, 1, 'dev7', 'g5e7fa38.d20240219') diff --git a/simulatedmicroscopy/image.py b/src/simulatedmicroscopy/image.py similarity index 100% rename from simulatedmicroscopy/image.py rename to src/simulatedmicroscopy/image.py diff --git a/simulatedmicroscopy/input.py b/src/simulatedmicroscopy/input.py similarity index 100% rename from simulatedmicroscopy/input.py rename to src/simulatedmicroscopy/input.py diff --git a/simulatedmicroscopy/particle.py b/src/simulatedmicroscopy/particle.py similarity index 100% rename from simulatedmicroscopy/particle.py rename to src/simulatedmicroscopy/particle.py diff --git a/simulatedmicroscopy/psf.py b/src/simulatedmicroscopy/psf.py similarity index 100% rename from simulatedmicroscopy/psf.py rename to src/simulatedmicroscopy/psf.py From fb7ddc9d604fe1139517ba5905e7f2242577410d Mon Sep 17 00:00:00 2001 From: Roy Hoitink Date: Mon, 19 Feb 2024 11:21:00 +0100 Subject: [PATCH 5/6] Updated github actions versions --- .bumpversion.cfg | 16 ---------------- .github/workflows/deploydocs.yml | 4 ++-- .github/workflows/pythonpackage.yml | 4 ++-- .gitignore | 1 + 4 files changed, 5 insertions(+), 20 deletions(-) delete mode 100644 .bumpversion.cfg diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index 30bbee5..0000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,16 +0,0 @@ -[bumpversion] -current_version = 1.4.0 -commit = True -tag = True - -[bumpversion:file:simulatedmicroscopy/__init__.py] -search = __version__ = "{current_version}" -replace = __version__ = "{new_version}" - -[bumpversion:file:setup.py] -search = version="{current_version}" -replace = version="{new_version}" - -[bumpversion:file:CITATION.cff] -search = version: "v{current_version}" -replace = version: "v{new_version}" diff --git a/.github/workflows/deploydocs.yml b/.github/workflows/deploydocs.yml index 680e06a..8d75c6f 100644 --- a/.github/workflows/deploydocs.yml +++ b/.github/workflows/deploydocs.yml @@ -12,9 +12,9 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python 3.x - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.10' architecture: 'x64' diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 43f6f65..cdee32c 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -12,9 +12,9 @@ jobs: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.gitignore b/.gitignore index 3145646..95bb79a 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ htmlcov site *.xyz *.ipynb +simulatedmicroscopy/_version.py From 761dbe2ca9ccb8508f709165f6760cb1154ec20d Mon Sep 17 00:00:00 2001 From: Roy Hoitink Date: Mon, 19 Feb 2024 11:27:22 +0100 Subject: [PATCH 6/6] Removed _version.py from git --- .gitignore | 2 +- src/simulatedmicroscopy/_version.py | 16 ---------------- 2 files changed, 1 insertion(+), 17 deletions(-) delete mode 100644 src/simulatedmicroscopy/_version.py diff --git a/.gitignore b/.gitignore index 95bb79a..e43210f 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,4 @@ htmlcov site *.xyz *.ipynb -simulatedmicroscopy/_version.py +src/simulatedmicroscopy/_version.py diff --git a/src/simulatedmicroscopy/_version.py b/src/simulatedmicroscopy/_version.py deleted file mode 100644 index 081efef..0000000 --- a/src/simulatedmicroscopy/_version.py +++ /dev/null @@ -1,16 +0,0 @@ -# file generated by setuptools_scm -# don't change, don't track in version control -TYPE_CHECKING = False -if TYPE_CHECKING: - from typing import Tuple, Union - VERSION_TUPLE = Tuple[Union[int, str], ...] -else: - VERSION_TUPLE = object - -version: str -__version__: str -__version_tuple__: VERSION_TUPLE -version_tuple: VERSION_TUPLE - -__version__ = version = '1.4.1.dev7+g5e7fa38.d20240219' -__version_tuple__ = version_tuple = (1, 4, 1, 'dev7', 'g5e7fa38.d20240219')