diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0dd35fc..937a908 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -49,6 +49,8 @@ jobs: python -m pip install --upgrade pip wheel 'setuptools<58' pip install -r requirements_dev.txt pip install -U codecov + - name: Print out pip freeze + run: pip freeze - name: Lint run: | pre-commit install @@ -87,14 +89,16 @@ jobs: run: echo "$IS_PR_TARGETING_MASTER" "$MASTER_PUSH" - name: Run tests # use temporary directory cleaned up after every job - run: pytest --basetemp={runner.temp} --cov=./ --cov-report xml --mpl --mpl-results-path=tests/results + run: pytest --basetemp=${{ runner.temp }} --cov=./ --cov-report xml --mpl --mpl-results-path=tests/results env: MPLBACKEND: Agg - name: Upload pytest test result artifacts on failure uses: actions/upload-artifact@v2 with: name: pytest-results-${{ matrix.python-version }} - path: tests/results + path: | + tests/results + ${{ runner.temp }}/test*current if: ${{ failure() }} - name: Upload coverage on success uses: codecov/codecov-action@v1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1ca88b2..096a961 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,6 +11,6 @@ repos: - id: requirements-txt-fixer - id: check-merge-conflict - repo: https://github.com/psf/black - rev: 19.3b0 + rev: 22.3.0 hooks: - id: black diff --git a/Makefile b/Makefile index 3d707f7..51c061b 100644 --- a/Makefile +++ b/Makefile @@ -72,11 +72,11 @@ test-without-figures: ## regenerate baseline figures regen-snapshot-figures: build-docker-test-image - docker run --rm -it -v $$(pwd):/src genetools-test pytest --mpl-generate-path=tests/baseline; + docker run --rm -it -v $$(pwd):/src genetools-test pytest --mpl-generate-path=tests/baseline --snapshot-update; ## regenerate saved test data (and baseline figures) regen-test-data: build-docker-test-image - docker run --rm -it -v $$(pwd):/src genetools-test pytest --mpl-generate-path=tests/baseline --regenerate-anndata; + docker run --rm -it -v $$(pwd):/src genetools-test pytest --mpl-generate-path=tests/baseline --snapshot-update --regenerate-anndata; coverage: ## check code coverage quickly with the default Python coverage run --source genetools -m pytest diff --git a/requirements_dev.txt b/requirements_dev.txt index 8b83ccb..a161067 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -5,16 +5,16 @@ flake8==3.7.8 h5py>=2.10.0 igraph>=0.9.9 louvain>=0.6.1 -matplotlib>=3.5.1 +matplotlib>=3.5.2 # numba requires numpy<21 numpy>=1.18.3,<1.21 pandas>=1.3.5 pip>=21.1.1 pre-commit>=2.15.0 -pytest==4.6.5 -pytest-cov==2.10.0 +pytest-cov>=2.10.0 pytest-mpl==0.11 pytest-runner==5.1 +pytest>=5.1.0,<8.0.0 recommonmark>=0.7.1 requests>=2.26.0 scanpy>=1.8.2 @@ -22,6 +22,7 @@ scikit-learn>=1.0.1 scipy>=1.4.1 seaborn>=0.11.2 sphinx>=4 +syrupy>=2.3.1 twine==1.14.0 watchdog==0.9.0 wheel==0.33.6 diff --git a/tests/__snapshots__/test_plots.ambr b/tests/__snapshots__/test_plots.ambr new file mode 100644 index 0000000..74fec27 --- /dev/null +++ b/tests/__snapshots__/test_plots.ambr @@ -0,0 +1,3 @@ +# name: test_pdf_deterministic_output + 'e8910b326f5ab726de325b29cf11f5a2' +# --- diff --git a/tests/test_plots.py b/tests/test_plots.py index f54d5d0..aa7c4a8 100644 --- a/tests/test_plots.py +++ b/tests/test_plots.py @@ -257,10 +257,17 @@ def test_wrap_labels_overrides_any_linebreaks_in_labels(): return fig -def test_pdf_deterministic_output(tmp_path): +def test_pdf_deterministic_output(tmp_path, snapshot): # Can't use snapshot_image here because pytest-mpl doesn't support PDF - # So we are doing our own snapshot test md5 checksum here + # So we are doing our own snapshot test md5 checksum here. + # + # Unlike pytest-mpl, which allows minute differences in the resulting image, + # the MD5 checksum relies on exact output from matplotlib, which is not guaranteed to be stable. + # The expected MD5 checksum will change based on matplotlib version unfortunately. + # We use syrupy to store the expected MD5 checksum. Regenerate with pytest --snapshot-update + # # This also allows us to test genetools.plots.savefig directly. + # fname = tmp_path / "test_pdf_determinstic_output.pdf" @@ -277,12 +284,10 @@ def get_md5(fname): with open(fname, "rb") as f: return hashlib.md5(f.read()).hexdigest() - # A hacky snapshot test: put expected md5sum here - expected_md5 = "4fae9ebe9cb8f837aed495fee12ca179" observed_md5 = get_md5(fname) assert ( - expected_md5 == observed_md5 - ), f"{fname} md5sum mismatch: got {observed_md5}, expected {expected_md5}" + observed_md5 == snapshot + ), f"{fname} md5sum mismatch: got {observed_md5}, expected {snapshot}" @snapshot_image diff --git a/tests/test_stats.py b/tests/test_stats.py index 00023d8..6207dcc 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -50,8 +50,7 @@ def test_normalize_cols(): def test_self_coclustering(): - """Coclustering sanity check: dataset against itself should always yield 100% - """ + """Coclustering sanity check: dataset against itself should always yield 100%""" cluster_ids = [1, 1, 2, 2] # 6 potential relationships in each dataset: 1-2, 1-3, 1-4, 2-3, 2-4, 3-4 # Actual coclustering in each dataset: 1-2, 3-4