feat(snapshots): add snapshot fixtures, remove pandas fixture (MODFLO…

…W-USGS#151)

* add numpy array snapshot extensions and fixtures with syrupy, moved from flopy
* remove use_pandas fixture, originally meant for use in flopy tests for pandas support but never used and probably not needed, as flopy use_pandas flags will be removed eventually
* todo: figure out how to support multi-array snapshots

README.md

@@ -56,6 +56,7 @@ Python3.8+, dependency-free, but pairs well with `pytest` and select plugins, e.
 
 - [`pytest-dotenv`](https://github.com/quiqua/pytest-dotenv)
 - [`pytest-xdist`](https://github.com/pytest-dev/pytest-xdist)
+- [`syrupy`](https://github.com/tophat/syrupy)
 
 ## Installation
 
@@ -65,7 +66,7 @@ Python3.8+, dependency-free, but pairs well with `pytest` and select plugins, e.
 pip install modflow-devtools
 ```
 
-Pytest, pytest plugins, and other optional dependencies can be installed with:
+Pytest, pytest plugins, and other testing-related dependencies can be installed with:
 
 ```shell
 pip install "modflow-devtools[test]"

autotest/__snapshots__/test_fixtures/test_readable_text_array_snapshot.txt

@@ -0,0 +1 @@
+[1.1 2.2 3.3]

autotest/__snapshots__/test_fixtures/test_text_array_snapshot.txt

@@ -0,0 +1,3 @@
+1.100000000000000089e+00
+2.200000000000000178e+00
+3.299999999999999822e+00

autotest/test_fixtures.py

@@ -2,11 +2,13 @@
 import platform
 from pathlib import Path
 
+import numpy as np
 import pytest
 from _pytest.config import ExitCode
 
 system = platform.system()
-proj_root = Path(__file__).parent.parent.parent.parent
+proj_root = Path(__file__).parents[1]
+module_path = Path(inspect.getmodulename(__file__))
 
 
 # test temporary directory fixtures
@@ -63,7 +65,7 @@ def test_class_scoped_tmpdir(self, class_tmpdir):
 def test_module_scoped_tmpdir(module_tmpdir):
     assert isinstance(module_tmpdir, Path)
     assert module_tmpdir.is_dir()
-    assert Path(inspect.getmodulename(__file__)).stem in module_tmpdir.name
+    assert module_path.stem in module_tmpdir.name
 
 
 def test_session_scoped_tmpdir(session_tmpdir):
@@ -269,41 +271,7 @@ def test_large_test_model(large_test_model):
     assert large_test_model.name == "mfsim.nam"
 
 
-# test pandas fixture
-
-test_pandas_fname = "pandas.txt"
-
-
-@pytest.mark.meta("test_pandas")
-def test_pandas_inner(function_tmpdir, use_pandas):
-    with open(function_tmpdir / test_pandas_fname, "w") as f:
-        f.write(str(use_pandas))
-
-
-@pytest.mark.parametrize("pandas", ["yes", "no", "random"])
-@pytest.mark.parametrize("arg", ["--pandas", "-P"])
-def test_pandas(pandas, arg, function_tmpdir):
-    inner_fn = test_pandas_inner.__name__
-    args = [
-        __file__,
-        "-v",
-        "-s",
-        "-k",
-        inner_fn,
-        arg,
-        pandas,
-        "--keep",
-        function_tmpdir,
-        "-M",
-        "test_pandas",
-    ]
-    assert pytest.main(args) == ExitCode.OK
-    res = open(next(function_tmpdir.rglob(test_pandas_fname))).readlines()[0]
-    assert res
-    if pandas == "yes":
-        assert "True" in res
-    elif pandas == "no":
-        assert "False" in res
+# test tabular data format fixture
 
 
 test_tabular_fname = "tabular.txt"
@@ -335,3 +303,63 @@ def test_tabular(tabular, arg, function_tmpdir):
     assert pytest.main(args) == ExitCode.OK
     res = open(next(function_tmpdir.rglob(test_tabular_fname))).readlines()[0]
     assert tabular == res
+
+
+# test snapshot fixtures
+
+
+snapshot_array = np.array([1.1, 2.2, 3.3])
+snapshots_path = proj_root / "autotest" / "__snapshots__"
+
+
+def test_binary_array_snapshot(array_snapshot):
+    assert array_snapshot == snapshot_array
+    snapshot_path = (
+        snapshots_path
+        / module_path.stem
+        / f"{inspect.currentframe().f_code.co_name}.npy"
+    )
+    assert snapshot_path.is_file()
+    assert np.allclose(np.load(snapshot_path), snapshot_array)
+
+
+# todo: reinstate if/when we support multiple arrays
+# def test_binary_array_snapshot_multi(array_snapshot):
+#     arrays = {"ascending": snapshot_array, "descending": np.flip(snapshot_array)}
+#     assert array_snapshot == arrays
+#     snapshot_path = (
+#         snapshots_path
+#         / module_path.stem
+#         / f"{inspect.currentframe().f_code.co_name}.npy"
+#     )
+#     assert snapshot_path.is_file()
+#     assert np.allclose(np.load(snapshot_path)["ascending"], snapshot_array)
+#     assert np.allclose(np.load(snapshot_path)["descending"], np.flip(snapshot_array))
+
+
+def test_text_array_snapshot(text_array_snapshot):
+    assert text_array_snapshot == snapshot_array
+    snapshot_path = (
+        snapshots_path
+        / module_path.stem
+        / f"{inspect.currentframe().f_code.co_name}.txt"
+    )
+    assert snapshot_path.is_file()
+    assert np.allclose(np.loadtxt(snapshot_path), snapshot_array)
+
+
+def test_readable_text_array_snapshot(readable_array_snapshot):
+    assert readable_array_snapshot == snapshot_array
+    snapshot_path = (
+        snapshots_path
+        / module_path.stem
+        / f"{inspect.currentframe().f_code.co_name}.txt"
+    )
+    assert snapshot_path.is_file()
+    assert np.allclose(
+        np.fromstring(
+            open(snapshot_path).readlines()[0].replace("[", "").replace("]", ""),
+            sep=" ",
+        ),
+        snapshot_array,
+    )

docs/md/fixtures.md

@@ -169,3 +169,13 @@ Model-loading fixtures use a set of utility functions to find and enumerate mode
 - `get_namefile_paths()`
 
 These functions are used internally in a `pytest_generate_tests` hook to implement the above model-parametrization fixtures. See `fixtures.py` and/or this project's test suite for usage examples.
+
+## Snapshot testing
+
+Snapshot testing is a form of regression testing in which a "snapshot" of the results of some computation is verified and captured by the developer to be compared against when tests are subsequently run. This is accomplished with [`syrupy`](https://github.com/tophat/syrupy), which provides a `snapshot` fixture overriding the equality operator to allow comparison with e.g. `snapshot == result`. A few custom fixtures for snapshots of NumPy arrays are also provided:
+
+- `array_snapshot`: saves an array in a binary file for compact storage, can be inspected programmatically with `np.load()`
+- `text_array_snapshot`: flattens an array and stores it in a text file, compromise between readability and disk usage
+- `readable_array_snapshot`: stores an array in a text file in its original shape, easy to inspect but largest on disk
+
+By default, tests run in comparison mode. This means a newly written test using any of the snapshot fixtures will fail until a snapshot is created. Snapshots can be created/updated by running pytest with the `--snapshot-update` flag.

modflow_devtools/fixtures.py

@@ -1,22 +1,106 @@
-import random
 from collections import OrderedDict
+from io import BytesIO, StringIO
 from itertools import groupby
 from os import PathLike, environ
 from pathlib import Path
 from shutil import copytree, rmtree
-from typing import Dict, List, Optional
+from typing import Dict, Generator, List, Optional
 
 from modflow_devtools.imports import import_optional_dependency
 from modflow_devtools.misc import get_namefile_paths, get_packages
 
+np = import_optional_dependency("numpy")
 pytest = import_optional_dependency("pytest")
+syrupy = import_optional_dependency("syrupy")
+
+# ruff: noqa: E402
+from syrupy.extensions.single_file import (
+    SingleFileSnapshotExtension,
+    WriteMode,
+)
+from syrupy.types import (
+    PropertyFilter,
+    PropertyMatcher,
+    SerializableData,
+    SerializedData,
+)
+
+# snapshot extensions
+
+
+class BinaryArrayExtension(SingleFileSnapshotExtension):
+    """
+    Binary snapshot of a NumPy array. Can be read back into NumPy with
+    .load(), preserving dtype and shape. This is the recommended array
+    snapshot approach if human-readability is not a necessity, as disk
+    space is minimized.
+    """
+
+    _write_mode = WriteMode.BINARY
+    _file_extension = "npy"
+
+    def serialize(
+        self,
+        data,
+        *,
+        exclude=None,
+        include=None,
+        matcher=None,
+    ):
+        buffer = BytesIO()
+        np.save(buffer, data)
+        return buffer.getvalue()
+
+
+class TextArrayExtension(SingleFileSnapshotExtension):
+    """
+    Text snapshot of a NumPy array. Flattens the array before writing.
+    Can be read back into NumPy with .loadtxt() assuming you know the
+    shape of the expected data and subsequently reshape it if needed.
+    """
+
+    _write_mode = WriteMode.TEXT
+    _file_extension = "txt"
+
+    def serialize(
+        self,
+        data: "SerializableData",
+        *,
+        exclude: Optional["PropertyFilter"] = None,
+        include: Optional["PropertyFilter"] = None,
+        matcher: Optional["PropertyMatcher"] = None,
+    ) -> "SerializedData":
+        buffer = StringIO()
+        np.savetxt(buffer, data.ravel())
+        return buffer.getvalue()
+
+
+class ReadableArrayExtension(SingleFileSnapshotExtension):
+    """
+    Human-readable snapshot of a NumPy array. Preserves array shape
+    at the expense of possible loss of precision (default 8 places)
+    and more difficulty loading into NumPy than TextArrayExtension.
+    """
+
+    _write_mode = WriteMode.TEXT
+    _file_extension = "txt"
+
+    def serialize(
+        self,
+        data: "SerializableData",
+        *,
+        exclude: Optional["PropertyFilter"] = None,
+        include: Optional["PropertyFilter"] = None,
+        matcher: Optional["PropertyMatcher"] = None,
+    ) -> "SerializedData":
+        return np.array2string(data, threshold=np.inf)
 
 
 # fixtures
 
 
 @pytest.fixture(scope="function")
-def function_tmpdir(tmpdir_factory, request) -> Path:
+def function_tmpdir(tmpdir_factory, request) -> Generator[Path, None, None]:
     node = request.node.name.replace("/", "_").replace("\\", "_").replace(":", "_")
     temp = Path(tmpdir_factory.mktemp(node))
     yield Path(temp)
@@ -37,7 +121,7 @@ def function_tmpdir(tmpdir_factory, request) -> Path:
 
 
 @pytest.fixture(scope="class")
-def class_tmpdir(tmpdir_factory, request) -> Path:
+def class_tmpdir(tmpdir_factory, request) -> Generator[Path, None, None]:
     assert (
         request.cls is not None
     ), "Class-scoped temp dir fixture must be used on class"
@@ -53,7 +137,7 @@ def class_tmpdir(tmpdir_factory, request) -> Path:
 
 
 @pytest.fixture(scope="module")
-def module_tmpdir(tmpdir_factory, request) -> Path:
+def module_tmpdir(tmpdir_factory, request) -> Generator[Path, None, None]:
     temp = Path(tmpdir_factory.mktemp(request.module.__name__))
     yield temp
 
@@ -66,7 +150,7 @@ def module_tmpdir(tmpdir_factory, request) -> Path:
 
 
 @pytest.fixture(scope="session")
-def session_tmpdir(tmpdir_factory, request) -> Path:
+def session_tmpdir(tmpdir_factory, request) -> Generator[Path, None, None]:
     temp = Path(tmpdir_factory.mktemp(request.config.rootpath.name))
     yield temp
 
@@ -85,26 +169,28 @@ def repos_path() -> Optional[Path]:
 
 
 @pytest.fixture
-def use_pandas(request):
-    pandas = request.config.option.PANDAS
-    if pandas == "yes":
-        return True
-    elif pandas == "no":
-        return False
-    elif pandas == "random":
-        return random.randint(0, 1) == 0
-    else:
-        raise ValueError(f"Unsupported value for --pandas: {pandas}")
-
-
-@pytest.fixture
-def tabular(request):
+def tabular(request) -> str:
     tab = request.config.option.TABULAR
     if tab not in ["raw", "recarray", "dataframe"]:
         raise ValueError(f"Unsupported value for --tabular: {tab}")
     return tab
 
 
+@pytest.fixture
+def array_snapshot(snapshot):
+    return snapshot.use_extension(BinaryArrayExtension)
+
+
+@pytest.fixture
+def text_array_snapshot(snapshot):
+    return snapshot.use_extension(TextArrayExtension)
+
+
+@pytest.fixture
+def readable_array_snapshot(snapshot):
+    return snapshot.use_extension(ReadableArrayExtension)
+
+
 # configuration hooks
 
 
@@ -168,16 +254,6 @@ def pytest_addoption(parser):
         help="Select a subset of packages to run.",
     )
 
-    parser.addoption(
-        "-P",
-        "--pandas",
-        action="store",
-        default="yes",
-        dest="PANDAS",
-        help="Indicates whether to use pandas, where multiple approaches "
-        "are available. Select 'yes', 'no', or 'random'.",
-    )
-
     parser.addoption(
         "-T",
         "--tabular",
@@ -399,5 +475,5 @@ def get_examples():
         metafunc.parametrize(
             key,
             [(name, nfps) for name, nfps in example_scenarios.items()],
-            ids=[name for name, ex in example_scenarios.items()],
+            ids=list(example_scenarios.keys()),
         )

pyproject.toml

@@ -54,11 +54,13 @@ test = [
     "meson!=0.63.0",
     "ninja",
     "numpy",
+    "pandas",
     "pytest!=8.1.0",
     "pytest-cov",
     "pytest-dotenv",
     "pytest-xdist",
-    "PyYaml"
+    "PyYaml",
+    "syrupy"
 ]
 docs = [
     "sphinx",