From 2dfe43dcf1e85a53668268c2c6fcb649d7a133bb Mon Sep 17 00:00:00 2001 From: w-bonelli Date: Mon, 13 May 2024 18:17:59 -0400 Subject: [PATCH 1/2] refactor(snapshots): move to separate module --- README.md | 2 +- .../test_binary_array_snapshot.npy | Bin .../test_readable_text_array_snapshot.txt | 0 .../test_text_array_snapshot.txt | 0 autotest/test_fixtures.py | 61 ---------- autotest/test_snapshots.py | 63 ++++++++++ docs/md/fixtures.md | 10 -- docs/md/snapshots.md | 17 +++ modflow_devtools/fixtures.py | 100 ---------------- modflow_devtools/snapshots.py | 108 ++++++++++++++++++ 10 files changed, 189 insertions(+), 172 deletions(-) rename autotest/__snapshots__/{test_fixtures => test_snapshots}/test_binary_array_snapshot.npy (100%) rename autotest/__snapshots__/{test_fixtures => test_snapshots}/test_readable_text_array_snapshot.txt (100%) rename autotest/__snapshots__/{test_fixtures => test_snapshots}/test_text_array_snapshot.txt (100%) create mode 100644 autotest/test_snapshots.py create mode 100644 docs/md/snapshots.md create mode 100644 modflow_devtools/snapshots.py diff --git a/README.md b/README.md index b81da4a..726b0f6 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ pip install "modflow-devtools[test]" To install from source and set up a development environment please see the [developer documentation](DEVELOPER.md). -To import `pytest` fixtures in a project consuming `modflow-devtools`, add the following to a `conftest.py` file: +To import `pytest` fixtures in a project consuming `modflow-devtools`, add the following to a test file or `conftest.py` file: ```python pytest_plugins = [ "modflow_devtools.fixtures" ] diff --git a/autotest/__snapshots__/test_fixtures/test_binary_array_snapshot.npy b/autotest/__snapshots__/test_snapshots/test_binary_array_snapshot.npy similarity index 100% rename from autotest/__snapshots__/test_fixtures/test_binary_array_snapshot.npy rename to autotest/__snapshots__/test_snapshots/test_binary_array_snapshot.npy diff --git a/autotest/__snapshots__/test_fixtures/test_readable_text_array_snapshot.txt b/autotest/__snapshots__/test_snapshots/test_readable_text_array_snapshot.txt similarity index 100% rename from autotest/__snapshots__/test_fixtures/test_readable_text_array_snapshot.txt rename to autotest/__snapshots__/test_snapshots/test_readable_text_array_snapshot.txt diff --git a/autotest/__snapshots__/test_fixtures/test_text_array_snapshot.txt b/autotest/__snapshots__/test_snapshots/test_text_array_snapshot.txt similarity index 100% rename from autotest/__snapshots__/test_fixtures/test_text_array_snapshot.txt rename to autotest/__snapshots__/test_snapshots/test_text_array_snapshot.txt diff --git a/autotest/test_fixtures.py b/autotest/test_fixtures.py index 600a34c..b9fa5b5 100644 --- a/autotest/test_fixtures.py +++ b/autotest/test_fixtures.py @@ -2,7 +2,6 @@ import platform from pathlib import Path -import numpy as np import pytest from _pytest.config import ExitCode @@ -303,63 +302,3 @@ def test_tabular(tabular, arg, function_tmpdir): assert pytest.main(args) == ExitCode.OK res = open(next(function_tmpdir.rglob(test_tabular_fname))).readlines()[0] assert tabular == res - - -# test snapshot fixtures - - -snapshot_array = np.array([1.1, 2.2, 3.3]) -snapshots_path = proj_root / "autotest" / "__snapshots__" - - -def test_binary_array_snapshot(array_snapshot): - assert array_snapshot == snapshot_array - snapshot_path = ( - snapshots_path - / module_path.stem - / f"{inspect.currentframe().f_code.co_name}.npy" - ) - assert snapshot_path.is_file() - assert np.allclose(np.load(snapshot_path), snapshot_array) - - -# todo: reinstate if/when we support multiple arrays -# def test_binary_array_snapshot_multi(array_snapshot): -# arrays = {"ascending": snapshot_array, "descending": np.flip(snapshot_array)} -# assert array_snapshot == arrays -# snapshot_path = ( -# snapshots_path -# / module_path.stem -# / f"{inspect.currentframe().f_code.co_name}.npy" -# ) -# assert snapshot_path.is_file() -# assert np.allclose(np.load(snapshot_path)["ascending"], snapshot_array) -# assert np.allclose(np.load(snapshot_path)["descending"], np.flip(snapshot_array)) - - -def test_text_array_snapshot(text_array_snapshot): - assert text_array_snapshot == snapshot_array - snapshot_path = ( - snapshots_path - / module_path.stem - / f"{inspect.currentframe().f_code.co_name}.txt" - ) - assert snapshot_path.is_file() - assert np.allclose(np.loadtxt(snapshot_path), snapshot_array) - - -def test_readable_text_array_snapshot(readable_array_snapshot): - assert readable_array_snapshot == snapshot_array - snapshot_path = ( - snapshots_path - / module_path.stem - / f"{inspect.currentframe().f_code.co_name}.txt" - ) - assert snapshot_path.is_file() - assert np.allclose( - np.fromstring( - open(snapshot_path).readlines()[0].replace("[", "").replace("]", ""), - sep=" ", - ), - snapshot_array, - ) diff --git a/autotest/test_snapshots.py b/autotest/test_snapshots.py new file mode 100644 index 0000000..04cabc9 --- /dev/null +++ b/autotest/test_snapshots.py @@ -0,0 +1,63 @@ +import inspect +from pathlib import Path + +import numpy as np + +proj_root = Path(__file__).parents[1] +module_path = Path(inspect.getmodulename(__file__)) +pytest_plugins = [ "modflow_devtools.snapshots" ] # activate snapshot fixtures +snapshot_array = np.array([1.1, 2.2, 3.3]) +snapshots_path = proj_root / "autotest" / "__snapshots__" + + +def test_binary_array_snapshot(array_snapshot): + assert array_snapshot == snapshot_array + snapshot_path = ( + snapshots_path + / module_path.stem + / f"{inspect.currentframe().f_code.co_name}.npy" + ) + assert snapshot_path.is_file() + assert np.allclose(np.load(snapshot_path), snapshot_array) + + +# todo: reinstate if/when we support multiple arrays +# def test_binary_array_snapshot_multi(array_snapshot): +# arrays = {"ascending": snapshot_array, "descending": np.flip(snapshot_array)} +# assert array_snapshot == arrays +# snapshot_path = ( +# snapshots_path +# / module_path.stem +# / f"{inspect.currentframe().f_code.co_name}.npy" +# ) +# assert snapshot_path.is_file() +# assert np.allclose(np.load(snapshot_path)["ascending"], snapshot_array) +# assert np.allclose(np.load(snapshot_path)["descending"], np.flip(snapshot_array)) + + +def test_text_array_snapshot(text_array_snapshot): + assert text_array_snapshot == snapshot_array + snapshot_path = ( + snapshots_path + / module_path.stem + / f"{inspect.currentframe().f_code.co_name}.txt" + ) + assert snapshot_path.is_file() + assert np.allclose(np.loadtxt(snapshot_path), snapshot_array) + + +def test_readable_text_array_snapshot(readable_array_snapshot): + assert readable_array_snapshot == snapshot_array + snapshot_path = ( + snapshots_path + / module_path.stem + / f"{inspect.currentframe().f_code.co_name}.txt" + ) + assert snapshot_path.is_file() + assert np.allclose( + np.fromstring( + open(snapshot_path).readlines()[0].replace("[", "").replace("]", ""), + sep=" ", + ), + snapshot_array, + ) diff --git a/docs/md/fixtures.md b/docs/md/fixtures.md index 613fd96..b847f10 100644 --- a/docs/md/fixtures.md +++ b/docs/md/fixtures.md @@ -169,13 +169,3 @@ Model-loading fixtures use a set of utility functions to find and enumerate mode - `get_namefile_paths()` These functions are used internally in a `pytest_generate_tests` hook to implement the above model-parametrization fixtures. See `fixtures.py` and/or this project's test suite for usage examples. - -## Snapshot testing - -Snapshot testing is a form of regression testing in which a "snapshot" of the results of some computation is verified and captured by the developer to be compared against when tests are subsequently run. This is accomplished with [`syrupy`](https://github.com/tophat/syrupy), which provides a `snapshot` fixture overriding the equality operator to allow comparison with e.g. `snapshot == result`. A few custom fixtures for snapshots of NumPy arrays are also provided: - -- `array_snapshot`: saves an array in a binary file for compact storage, can be inspected programmatically with `np.load()` -- `text_array_snapshot`: flattens an array and stores it in a text file, compromise between readability and disk usage -- `readable_array_snapshot`: stores an array in a text file in its original shape, easy to inspect but largest on disk - -By default, tests run in comparison mode. This means a newly written test using any of the snapshot fixtures will fail until a snapshot is created. Snapshots can be created/updated by running pytest with the `--snapshot-update` flag. \ No newline at end of file diff --git a/docs/md/snapshots.md b/docs/md/snapshots.md new file mode 100644 index 0000000..0254617 --- /dev/null +++ b/docs/md/snapshots.md @@ -0,0 +1,17 @@ +# Snapshot testing + +Snapshot testing is a form of regression testing in which a "snapshot" of the results of some computation is verified and captured by the developer to be compared against when tests are subsequently run. This is accomplished with [`syrupy`](https://github.com/tophat/syrupy), which provides a `snapshot` fixture overriding the equality operator to allow comparison with e.g. `snapshot == result`. A few custom fixtures for snapshots of NumPy arrays are also provided: + +- `array_snapshot`: saves an array in a binary file for compact storage, can be inspected programmatically with `np.load()` +- `text_array_snapshot`: flattens an array and stores it in a text file, compromise between readability and disk usage +- `readable_array_snapshot`: stores an array in a text file in its original shape, easy to inspect but largest on disk + +By default, tests run in comparison mode. This means a newly written test using any of the snapshot fixtures will fail until a snapshot is created. Snapshots can be created/updated by running pytest with the `--snapshot-update` flag. + +## Using snapshot fixtures + +To use snapshot fixtures, add the following line to a test file or `conftest.py` file: + +```python +pytest_plugins = [ "modflow_devtools.snapshots" ] +``` \ No newline at end of file diff --git a/modflow_devtools/fixtures.py b/modflow_devtools/fixtures.py index 2d37328..963fa4e 100644 --- a/modflow_devtools/fixtures.py +++ b/modflow_devtools/fixtures.py @@ -1,5 +1,4 @@ from collections import OrderedDict -from io import BytesIO, StringIO from itertools import groupby from os import PathLike, environ from pathlib import Path @@ -9,91 +8,7 @@ from modflow_devtools.imports import import_optional_dependency from modflow_devtools.misc import get_namefile_paths, get_packages -np = import_optional_dependency("numpy") pytest = import_optional_dependency("pytest") -syrupy = import_optional_dependency("syrupy") - -# ruff: noqa: E402 -from syrupy.extensions.single_file import ( - SingleFileSnapshotExtension, - WriteMode, -) -from syrupy.types import ( - PropertyFilter, - PropertyMatcher, - SerializableData, - SerializedData, -) - -# snapshot extensions - - -class BinaryArrayExtension(SingleFileSnapshotExtension): - """ - Binary snapshot of a NumPy array. Can be read back into NumPy with - .load(), preserving dtype and shape. This is the recommended array - snapshot approach if human-readability is not a necessity, as disk - space is minimized. - """ - - _write_mode = WriteMode.BINARY - _file_extension = "npy" - - def serialize( - self, - data, - *, - exclude=None, - include=None, - matcher=None, - ): - buffer = BytesIO() - np.save(buffer, data) - return buffer.getvalue() - - -class TextArrayExtension(SingleFileSnapshotExtension): - """ - Text snapshot of a NumPy array. Flattens the array before writing. - Can be read back into NumPy with .loadtxt() assuming you know the - shape of the expected data and subsequently reshape it if needed. - """ - - _write_mode = WriteMode.TEXT - _file_extension = "txt" - - def serialize( - self, - data: "SerializableData", - *, - exclude: Optional["PropertyFilter"] = None, - include: Optional["PropertyFilter"] = None, - matcher: Optional["PropertyMatcher"] = None, - ) -> "SerializedData": - buffer = StringIO() - np.savetxt(buffer, data.ravel()) - return buffer.getvalue() - - -class ReadableArrayExtension(SingleFileSnapshotExtension): - """ - Human-readable snapshot of a NumPy array. Preserves array shape - at the expense of possible loss of precision (default 8 places) - and more difficulty loading into NumPy than TextArrayExtension. - """ - - _write_mode = WriteMode.TEXT - _file_extension = "txt" - - def serialize( - self, - data: "SerializableData", - *, - exclude: Optional["PropertyFilter"] = None, - include: Optional["PropertyFilter"] = None, - matcher: Optional["PropertyMatcher"] = None, - ) -> "SerializedData": - return np.array2string(data, threshold=np.inf) # fixtures @@ -176,21 +91,6 @@ def tabular(request) -> str: return tab -@pytest.fixture -def array_snapshot(snapshot): - return snapshot.use_extension(BinaryArrayExtension) - - -@pytest.fixture -def text_array_snapshot(snapshot): - return snapshot.use_extension(TextArrayExtension) - - -@pytest.fixture -def readable_array_snapshot(snapshot): - return snapshot.use_extension(ReadableArrayExtension) - - # configuration hooks diff --git a/modflow_devtools/snapshots.py b/modflow_devtools/snapshots.py new file mode 100644 index 0000000..eed8776 --- /dev/null +++ b/modflow_devtools/snapshots.py @@ -0,0 +1,108 @@ +from io import BytesIO, StringIO +from typing import Optional + +from modflow_devtools.imports import import_optional_dependency + +np = import_optional_dependency("numpy") +pytest = import_optional_dependency("pytest") +syrupy = import_optional_dependency("syrupy") + +# ruff: noqa: E402 +from syrupy.extensions.single_file import ( + SingleFileSnapshotExtension, + WriteMode, +) +from syrupy.types import ( + PropertyFilter, + PropertyMatcher, + SerializableData, + SerializedData, +) + +# extension classes + + +class BinaryArrayExtension(SingleFileSnapshotExtension): + """ + Binary snapshot of a NumPy array. Can be read back into NumPy with + .load(), preserving dtype and shape. This is the recommended array + snapshot approach if human-readability is not a necessity, as disk + space is minimized. + """ + + _write_mode = WriteMode.BINARY + _file_extension = "npy" + + def serialize( + self, + data, + *, + exclude=None, + include=None, + matcher=None, + ): + buffer = BytesIO() + np.save(buffer, data) + return buffer.getvalue() + + +class TextArrayExtension(SingleFileSnapshotExtension): + """ + Text snapshot of a NumPy array. Flattens the array before writing. + Can be read back into NumPy with .loadtxt() assuming you know the + shape of the expected data and subsequently reshape it if needed. + """ + + _write_mode = WriteMode.TEXT + _file_extension = "txt" + + def serialize( + self, + data: "SerializableData", + *, + exclude: Optional["PropertyFilter"] = None, + include: Optional["PropertyFilter"] = None, + matcher: Optional["PropertyMatcher"] = None, + ) -> "SerializedData": + buffer = StringIO() + np.savetxt(buffer, data.ravel()) + return buffer.getvalue() + + +class ReadableArrayExtension(SingleFileSnapshotExtension): + """ + Human-readable snapshot of a NumPy array. Preserves array shape + at the expense of possible loss of precision (default 8 places) + and more difficulty loading into NumPy than TextArrayExtension. + """ + + _write_mode = WriteMode.TEXT + _file_extension = "txt" + + def serialize( + self, + data: "SerializableData", + *, + exclude: Optional["PropertyFilter"] = None, + include: Optional["PropertyFilter"] = None, + matcher: Optional["PropertyMatcher"] = None, + ) -> "SerializedData": + return np.array2string(data, threshold=np.inf) + + +# fixtures + + +@pytest.fixture +def array_snapshot(snapshot): + return snapshot.use_extension(BinaryArrayExtension) + + +@pytest.fixture +def text_array_snapshot(snapshot): + return snapshot.use_extension(TextArrayExtension) + + +@pytest.fixture +def readable_array_snapshot(snapshot): + return snapshot.use_extension(ReadableArrayExtension) From 93726360d472ff0d39eb8efe3a84c7568e27fdd2 Mon Sep 17 00:00:00 2001 From: w-bonelli Date: Mon, 13 May 2024 18:23:31 -0400 Subject: [PATCH 2/2] remove outdated lint script --- .github/workflows/release.yml | 1 - scripts/lint.py | 21 --------------------- 2 files changed, 22 deletions(-) delete mode 100644 scripts/lint.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 890bdaf..1ff71fc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -45,7 +45,6 @@ jobs: ref="${{ github.ref_name }}" version="${ref#"v"}" python scripts/update_version.py -v "$version" - python scripts/lint.py python -c "import modflow_devtools; print('Version: ', modflow_devtools.__version__)" echo "version=$version" >> $GITHUB_OUTPUT diff --git a/scripts/lint.py b/scripts/lint.py deleted file mode 100644 index 315500a..0000000 --- a/scripts/lint.py +++ /dev/null @@ -1,21 +0,0 @@ -import os - -try: - import isort - - print(f"isort version: {isort.__version__}") -except ModuleNotFoundError: - print("isort not installed\n\tInstall using pip install isort") - -try: - import black - - print(f"black version: {black.__version__}") -except ModuleNotFoundError: - print("black not installed\n\tInstall using pip install black") - -print("running isort...") -os.system("isort -v .") - -print("running black...") -os.system("black -v .")