Update CI and dependencies after 1.5 years

.github/dependabot.yml

@@ -14,5 +14,6 @@ updates:
   # Maintain dependencies for GitHub Actions
   - package-ecosystem: "github-actions"
     directory: "/"
+    target-branch: "develop"
     schedule:
       interval: "daily"

.github/release-drafter-config.yml

@@ -0,0 +1,4 @@
+template: |
+  ## Changes (merged PRs)
+
+  $CHANGES

.github/workflows/ci.yaml

@@ -17,18 +17,18 @@ jobs:
       # don't abort all other jobs
       fail-fast: false
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        python-version: [3.8, 3.9]
     steps:
       - name: Checkout
         uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Get Python version
         run: python --version
       - name: Cache pip
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           # This path is specific to Ubuntu
           path: ~/.cache/pip
@@ -38,7 +38,7 @@ jobs:
             ${{ runner.os }}-pip-
             ${{ runner.os }}-
       - name: Cache precommit
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: ~/.cache/pre-commit
           # Look to see if there is a cache hit for the corresponding requirements file
@@ -93,15 +93,15 @@ jobs:
         env:
           MPLBACKEND: Agg
       - name: Upload pytest test result artifacts on failure
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: pytest-results-${{ matrix.python-version }}
           path: |
             tests/results
             ${{ runner.temp }}/test*current
         if: ${{ failure() }}
       - name: Upload coverage on success
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v4
         if: ${{ success() }}
 
   docs:
@@ -112,7 +112,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
       - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: 3.8
       - name: Install dependencies
@@ -132,6 +132,7 @@ jobs:
           args: deploy --dir="docs/_build/html"
         timeout-minutes: 5
       - name: deploy prod docs to netlify
+        # TODO: Switch to a production "environment" like in other projects
         if: ${{ env.MASTER_PUSH == 'true' }}
         uses: netlify/actions/cli@master
         env:
@@ -150,7 +151,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
       - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: 3.8
       - name: Install dependencies
@@ -177,3 +178,57 @@ jobs:
         with:
           user: __token__
           password: ${{ secrets.PYPI_API_TOKEN }}
+
+  make_github_tag_and_release:
+    needs: pypi
+
+    # Only even attempt using the environment if we are going to be able to
+    # Unfortunately env is not defined here, so can't use env.MASTER_PUSH directly (https://docs.github.com/en/actions/learn-github-actions/contexts#context-availability)
+    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+
+    permissions:
+      # write permission is required to create a github release
+      contents: write
+
+      # write permission is required for autolabeler
+      # otherwise, read permission is required at least
+      pull-requests: read # write
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Get new version
+        id: get-version
+        # Load current version (and a tag name with "v" prefix) into a step output
+        run: |
+          RAW_VERSION=$(python setup.py --version)
+          echo "TAG=v$RAW_VERSION" >> $GITHUB_OUTPUT
+          echo "VERSION=$RAW_VERSION" >> $GITHUB_OUTPUT
+      - name: Echo version for debug
+        run: echo "The new version is ${{ steps.get-version.outputs.VERSION }}, tag ${{ steps.get-version.outputs.TAG }}"
+
+
+      - name: Publish the release notes and tag new version, or drafts release notes as PRs merge into master
+        # This step succeeds even when release-drafter internally fails with an HttpError.
+        uses: release-drafter/release-drafter@v6
+        id: release_drafter
+        with:
+          config-name: release-drafter-config.yml
+          disable-autolabeler: true
+          publish: true # revert to this if we retry draft releases: ${{ env.MASTER_PUSH == 'true' }}
+          tag: ${{ steps.get-version.outputs.TAG }}
+          version: ${{ steps.get-version.outputs.VERSION }}
+          commitish: master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Error if Release Drafter threw errors but still exited successfully
+        # Detect the situation described above
+        if: toJSON(steps.release_drafter.outputs) == '{}'
+        # Error out but provide error message (https://stackoverflow.com/a/74229789/130164)
+        run: |
+          echo "::error Release drafter step failed above."
+          exit 1

.pre-commit-config.yaml

@@ -1,16 +1,21 @@
-# See https://pre-commit.com for more information
-# See https://pre-commit.com/hooks.html for more hooks
 repos:
--   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.4.0
-    hooks:
-    -   id: trailing-whitespace
-    -   id: end-of-file-fixer
-    -   id: check-yaml
-    -   id: check-added-large-files
-    -   id: requirements-txt-fixer
-    -   id: check-merge-conflict
--   repo: https://github.com/psf/black
-    rev: 22.3.0
-    hooks:
-    -   id: black
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v2.4.0
+  hooks:
+    - id: trailing-whitespace
+    - id: end-of-file-fixer
+    - id: check-yaml
+    - id: check-added-large-files
+      args: ['--maxkb=1000']
+    - id: requirements-txt-fixer
+    - id: check-merge-conflict
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.2.2
+  hooks:
+    # Run the linter.
+    - id: ruff
+      types_or: [ python, pyi, jupyter ]
+      args: [ --fix ]
+    # Run the formatter.
+    - id: ruff-format
+      types_or: [ python, pyi, jupyter ]

.ruff.toml

@@ -0,0 +1,2 @@
+# In addition to the standard set of exclusions
+extend-exclude = ["docs"]

Dockerfile

@@ -1,5 +1,5 @@
 # use debian-based python image to match Github Actions matplotlib outputs
-FROM python:3.8
+FROM python:3.9
 
 RUN mkdir /src
 WORKDIR /src

Makefile

@@ -63,7 +63,7 @@ build-docker-test-image: requirements_dev.txt
 # tests/results/tmp will be used for basetemp - can't use tests/results because then mpl results will be cleared out.
 test: build-docker-test-image
 	mkdir -p tests/results/tmp
-	docker run --rm -it -v $$(pwd):/src genetools-test pytest --cov=./ --cov-report term --cov-report xml --mpl --mpl-results-path=tests/results --basetemp=tests/results/tmp -vv;
+	docker run --rm -it -v $$(pwd):/src genetools-test pytest --cov=./ --cov-report term --cov-report xml --run-snapshots --mpl --mpl-results-path=tests/results --basetemp=tests/results/tmp -vv;
 
 # run tests locally, without docker, therefore omitting the snapshot tests
 test-without-figures:
@@ -72,11 +72,11 @@ test-without-figures:
 
 ## regenerate baseline figures
 regen-snapshot-figures: build-docker-test-image
-	docker run --rm -it -v $$(pwd):/src genetools-test pytest --mpl-generate-path=tests/baseline --snapshot-update;
+	docker run --rm -it -v $$(pwd):/src genetools-test pytest --mpl-generate-path=tests/baseline --run-snapshots --snapshot-update;
 
 ## regenerate saved test data (and baseline figures)
 regen-test-data: build-docker-test-image
-	docker run --rm -it -v $$(pwd):/src genetools-test pytest --mpl-generate-path=tests/baseline --snapshot-update --regenerate-anndata;
+	docker run --rm -it -v $$(pwd):/src genetools-test pytest --mpl-generate-path=tests/baseline --run-snapshots --snapshot-update --regenerate-anndata;
 
 coverage: ## check code coverage quickly with the default Python
 	coverage run --source genetools -m pytest
@@ -91,17 +91,3 @@ docs: ## generate Sphinx HTML documentation, including API docs
 	$(MAKE) -C docs clean
 	$(MAKE) -C docs html
 	$(BROWSER) docs/_build/html/index.html
-
-servedocs: docs ## compile the docs watching for changes
-	watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D .
-
-release: dist ## package and upload a release
-	twine upload dist/*
-
-dist: clean ## builds source and wheel package
-	python setup.py sdist
-	python setup.py bdist_wheel
-	ls -l dist
-
-install: clean ## install the package to the active Python's site-packages
-	python setup.py install

genetools/helpers.py

@@ -52,7 +52,7 @@ def merge_into_left(left, right, **kwargs):
         right_index=True,
         sort=False,
         validate="1:1",
-        **kwargs
+        **kwargs,
     )
     # TODO: asserts are stripped away when code is optimized; replace with if not, raise ValueError('message')
     assert df.shape[0] == df1.shape[0]

genetools/plots.py

@@ -1,5 +1,4 @@
 import os
-import numpy as np
 import matplotlib
 import matplotlib.pyplot as plt
 import pandas as pd
@@ -280,7 +279,7 @@ def scatterplot(
                 borderpad=0,
             )
 
-            colorbar = fig.colorbar(scattered_object, cax=colorbar_ax)
+            fig.colorbar(scattered_object, cax=colorbar_ax)
             if legend_title is not None:
                 colorbar_ax.set_title(legend_title)
 
@@ -450,7 +449,7 @@ def stacked_bar_plot(
 
     if normalize:
         # Normalize values to sum to 1 per row
-        plot_df[value_key] = plot_df.groupby(index_key, sort=False)[value_key].apply(
+        plot_df[value_key] = plot_df.groupby(index_key)[value_key].transform(
             lambda g: g / g.sum()
         )
 
@@ -465,7 +464,7 @@ def stacked_bar_plot(
     # Accumulate value with every subsequent box/hue as we go across each index/row
     # These will become row-level "left offsets" for each hue
     cum_value_key = value_key + "_cumulative_value"
-    plot_df[cum_value_key] = plot_df.groupby(index_key, sort=False)[value_key].cumsum()
+    plot_df[cum_value_key] = plot_df.groupby(index_key)[value_key].cumsum()
 
     # create colors
     if palette is None:

genetools/scanpy_helpers.py

@@ -2,6 +2,7 @@
 
 from . import stats
 
+
 # TODO: enable speeding this up by using highly variable genes only?
 def find_all_markers(
     adata,
@@ -32,8 +33,6 @@ def find_all_markers(
     :rtype: pandas.DataFrame
     """
 
-    import scipy
-    import numpy as np
     import pandas as pd
     import scanpy as sc
 

genetools/stats.py

@@ -167,7 +167,8 @@ def get_coclustering_indices(cells_by_cells):
         # extract non-diagonal and true
         out = np.array(
             np.where(
-                ~np.eye(cells_by_cells.shape[0], dtype=bool) & (cells_by_cells == True)
+                # Ignore Ruff E712 that asks us to change to "cells_by_cells is True"
+                ~np.eye(cells_by_cells.shape[0], dtype=bool) & (cells_by_cells == True)  # noqa: E712
             )
         ).T
         assert out.shape[1] == 2, "each row should have two cell IDs"

requirements_dev.txt

@@ -1,28 +1,25 @@
 anndata>=0.7.8
-bump2version==0.5.11
-coverage==4.5.4
-flake8==3.7.8
+bump2version>=1.0.1
+coverage>=7.2.2
 h5py>=2.10.0
 igraph>=0.9.9
 louvain>=0.6.1
-matplotlib==3.5.2
-# numba requires numpy<21
-numpy>=1.18.3,<1.21
-pandas>=1.3.5
+matplotlib>=3.5.2
+numpy>=1.24.3
+pandas >= 1.5.0
 pip>=21.1.1
 pre-commit>=2.15.0
-pytest-cov>=2.10.0
+pytest-cov>=4.0.0
 pytest-mpl==0.11
-pytest-runner==5.1
-pytest>=5.1.0,<8.0.0
+pytest-runner>=5.1
+pytest>=7.0.0
 recommonmark>=0.7.1
 requests>=2.26.0
+ruff>=0.2.2
 scanpy>=1.8.2
 scikit-learn>=1.0.1
 scipy>=1.4.1
 seaborn>=0.11.2
-sphinx>=4
+sphinx>=6
 syrupy>=2.3.1
-twine==1.14.0
-watchdog==0.9.0
-wheel==0.33.6
+wheel>=0.33.6

setup.cfg

@@ -1,9 +1,6 @@
 [bdist_wheel]
 universal = 1
 
-[flake8]
-exclude = docs
-
 [aliases]
 test = pytest
 

setup.py

@@ -19,14 +19,13 @@
 setup(
     author="Maxim Zaslavsky",
     author_email="maxim@maximz.com",
-    python_requires=">=3.7",
+    python_requires=">=3.8",
     classifiers=[
         "Development Status :: 2 - Pre-Alpha",
         "Intended Audience :: Developers",
         "License :: OSI Approved :: MIT License",
         "Natural Language :: English",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
     ],
     description="General genetics/genomics utilities.",

tests/__snapshots__/test_plots.ambr

@@ -1,3 +1,4 @@
+# serializer version: 1
 # name: test_pdf_deterministic_output
-  'e8910b326f5ab726de325b29cf11f5a2'
+  '6bd668f3827318d3b5e1a3c61f52e9c4'
 # ---