chore: add new nox sessions to deal with requirements and venvs (#13)

Add: - New `venv-3xx` nox session to automatically create and setup development environments - New `requirements` nox session to automatically freeze and synchronize versions of requirements for development and testing across tools - `requirements` folder with different sets of requirements for development Change: - Update development instructions - Update documentation-related nox sessions - Update minimum required dependency for `dace` Remove: - Old `requirements-dev.txt` file is now a preset inside the `requirements` folder
GridTools · Jun 17, 2024 · ca03fbd · ca03fbd
1 parent a1355a2
commit ca03fbd
Show file tree

Hide file tree

Showing 15 changed files with 676 additions and 52 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -50,7 +50,7 @@ jobs:
         allow-prereleases: true
 
     - name: Install requirementes
-      run: python -m pip install -r requirements-dev.txt
+      run: python -m pip install -r requirements/dev.txt
 
     - name: Install package
       run: python -m pip install .

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -52,11 +52,10 @@ repos:
   - id: debug-statements
   - id: end-of-file-fixer
   - id: mixed-line-ending
-  - id: requirements-txt-fixer
   - id: trailing-whitespace
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.4.8
+  rev: v0.4.9
   hooks:
   - id: ruff
     args: [--fix, --show-fixes]
@@ -69,10 +68,11 @@ repos:
     files: src|tests
     args: [--no-install-types]
     additional_dependencies:
-    - dace==0.15.1
-    - jax[cpu]==0.4.28
-    - numpy==1.26.4
-    - pytest==8.2.1
+    - dace==0.16
+    - jax[cpu]==0.4.29
+    - numpy==2.0.0
+    - pytest==8.2.2
+    - typing-extensions==4.12.2
 - repo: https://github.com/codespell-project/codespell
   rev: v2.3.0
   hooks:

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -9,6 +9,7 @@ The fastest way to start with development is to use nox. If you don't have nox,
 To use, run `nox`. This will lint and test using every installed version of Python on your system, skipping ones that are not installed. You can also run specific jobs:
 
 ```console
+$ nox -s venv-3.10  # (or venv-3.11, or venv-3.12) Setup a fully working development environment
 $ nox -s lint  # Lint only
 $ nox -s tests  # Python tests
 $ nox -s docs -- --serve  # Build and serve the docs
@@ -25,16 +26,16 @@ You can set up a development environment by running:
 python3 -m venv .venv
 source ./.venv/bin/activate
 pip install --upgrade pip setuptools wheel
-pip install -r requirements-dev.txt
+pip install -r requirements/dev.txt
 pip install -v -e .
 ```
 
-If you have the [Python Launcher for Unix](https://github.com/brettcannon/python-launcher), you can instead do:
+Or, if you have the [Python Launcher for Unix](https://github.com/brettcannon/python-launcher), you could do:
 
 ```bash
 py -m venv .venv
 py -m pip install --upgrade pip setuptools wheel
-py -m pip install -r requirements-dev.txt
+py -m pip install -r requirements/dev.txt
 py -m pip install -v -e .
 ```
 
@@ -43,7 +44,7 @@ py -m pip install -v -e .
 You should prepare pre-commit, which will help you by checking that commits pass required checks:
 
 ```bash
-pip install pre-commit # or brew install pre-commit on macOS
+pipx install pre-commit # or brew install pre-commit on macOS
 pre-commit install # Will install a pre-commit hook into the git repo
 ```
 

diff --git a/noxfile.py b/noxfile.py
@@ -1,36 +1,155 @@
+"""Nox session definitions."""
+
 from __future__ import annotations
 
 import argparse
+import pathlib
+import re
 import shutil
-from pathlib import Path
 
 import nox
 
 
-DIR = Path(__file__).parent.resolve()
-
 nox.needs_version = ">=2024.3.2"
-nox.options.sessions = ["lint", "pylint", "tests"]
+nox.options.sessions = ["lint", "tests"]
 nox.options.default_venv_backend = "uv|virtualenv"
 
 
-@nox.session
+ROOT_DIR = pathlib.Path(__file__).parent.resolve()
+DEFAULT_DEV_VENV_PATH = ROOT_DIR / ".venv"
+
+
+def load_from_frozen_requirements(filename: str) -> dict[str, str]:
+    requirements = {}
+    with pathlib.Path(filename).open(encoding="locale") as f:
+        for raw_line in f:
+            if (end := raw_line.find("#")) != -1:
+                raw_line = raw_line[:end]  # noqa: PLW2901 [redefined-loop-name]
+            line = raw_line.strip()
+            if line and not line.startswith("-"):
+                m = re.match(r"^([^=]*)\s*([^;]*)\s*;?\s*(.*)$", line)
+                if m:
+                    requirements[m[1]] = m[2]
+
+    return requirements
+
+
+REQUIREMENTS = load_from_frozen_requirements(ROOT_DIR / "requirements" / "dev.txt")
+
+
+@nox.session(python="3.10")
 def lint(session: nox.Session) -> None:
-    """Run the linter."""
+    """Run the linter (pre-commit)."""
     session.install("pre-commit")
     session.run("pre-commit", "run", "--all-files", "--show-diff-on-failure", *session.posargs)
 
 
 @nox.session
 def tests(session: nox.Session) -> None:
     """Run the unit and regular tests."""
-    session.install(".[test]")
+    session.install("-e", ".", "-r", "requirements/dev.txt")
     session.run("pytest", *session.posargs)
 
 
+@nox.session(python=["3.10", "3.11", "3.12"])
+def venv(session: nox.Session) -> None:
+    """
+    Sets up a Python development environment. Use as: `nox -s venv-3.xx -- [req_preset] [dest_path]
+
+    req_preset: The requirements file to use as 'requirements/{req_preset}.txt'.
+        Default: 'dev'
+    dest_path (optional): The path to the virtualenv to create.
+        Default: '.venv-{3.xx}-{req_preset}'
+
+    This session will:
+    - Create a python virtualenv for the session
+    - Install the `virtualenv` cli tool into this environment
+    - Use `virtualenv` to create a project virtual environment
+    - Invoke the python interpreter from the created project environment
+      to install the project and all it's development dependencies.
+    """  # noqa: W505 [doc-line-too-long]
+    req_preset = "dev"
+    venv_path = None
+    virtualenv_args = []
+    if session.posargs:
+        req_preset, *more_pos_args = session.posargs
+        if more_pos_args:
+            venv_path, *_ = more_pos_args
+    if not venv_path:
+        venv_path = f"{DEFAULT_DEV_VENV_PATH}-{session.python}-{req_preset}"
+    venv_path = pathlib.Path(venv_path).resolve()
+
+    if not venv_path.exists():
+        print(f"Creating virtualenv at '{venv_path}' (options: {virtualenv_args})...")
+        session.install("virtualenv")
+        session.run("virtualenv", venv_path, silent=True)
+    elif venv_path.exists():
+        assert (
+            venv_path.is_dir() and (venv_path / "bin" / f"python{session.python}").exists
+        ), f"'{venv_path}' path already exists but is not a virtualenv with python{session.python}."
+        print(f"'{venv_path}' path already exists. Skipping virtualenv creation...")
+
+    python_path = venv_path / "bin" / "python"
+    requirements_file = f"requirements/{req_preset}.txt"
+
+    # Use the venv's interpreter to install the project along with
+    # all it's dev dependencies, this ensures it's installed in the right way
+    print(f"Setting up development environment from '{requirements_file}'...")
+    session.run(
+        python_path,
+        "-m",
+        "pip",
+        "install",
+        "-r",
+        requirements_file,
+        "-e.",
+        external=True,
+    )
+
+
+@nox.session(reuse_venv=True)
+def requirements(session: nox.Session) -> None:
+    """Freeze requirements files from project specification and synchronize versions across tools."""  # noqa: W505 [doc-line-too-long]
+    requirements_path = ROOT_DIR / "requirements"
+    req_sync_tool = requirements_path / "sync_tool.py"
+
+    dependencies = ["pre-commit"] + nox.project.load_toml(req_sync_tool)["dependencies"]
+    session.install(*dependencies)
+    session.install("pip-compile-multi")
+
+    session.run("python", req_sync_tool, "pull")
+    session.run("pip-compile-multi", "-g", "--skip-constraints")
+    session.run("python", req_sync_tool, "push")
+
+    session.run("pre-commit", "run", "--files", ".pre-commit-config.yaml", success_codes=[0, 1])
+
+
 @nox.session(reuse_venv=True)
 def docs(session: nox.Session) -> None:
-    """Build the docs. Pass "--serve" to serve. Pass "-b linkcheck" to check links."""
+    """Regenerate and build all API and user docs."""
+    session.notify("api_docs")
+    session.notify("user_docs", posargs=session.posargs)
+
+
+@nox.session(reuse_venv=True)
+def api_docs(session: nox.Session) -> None:
+    """Build (regenerate) API docs."""
+    session.install(f"sphinx=={REQUIREMENTS['sphinx']}")
+    session.chdir("docs")
+    session.run(
+        "sphinx-apidoc",
+        "-o",
+        "api/",
+        "--module-first",
+        "--no-toc",
+        "--force",
+        "../src/jace",
+    )
+
+
+@nox.session(reuse_venv=True)
+def user_docs(session: nox.Session) -> None:
+    """Build the user docs. Pass "--serve" to serve. Pass "-b linkcheck" to check links."""  # noqa: W505 [doc-line-too-long]
     parser = argparse.ArgumentParser()
     parser.add_argument("--serve", action="store_true", help="Serve after building")
     parser.add_argument("-b", dest="builder", default="html", help="Build target (default: html)")
@@ -40,8 +159,7 @@ def docs(session: nox.Session) -> None:
         session.error("Must not specify non-HTML builder with --serve")
 
     extra_installs = ["sphinx-autobuild"] if args.serve else []
-
-    session.install("-e.[docs]", *extra_installs)
+    session.install("-e", ".", "-r", "requirements/dev.txt", *extra_installs)
     session.chdir("docs")
 
     if args.builder == "linkcheck":
@@ -63,28 +181,12 @@ def docs(session: nox.Session) -> None:
         session.run("sphinx-build", "--keep-going", *shared_args)
 
 
-@nox.session
-def build_api_docs(session: nox.Session) -> None:
-    """Build (regenerate) API docs."""
-    session.install("sphinx")
-    session.chdir("docs")
-    session.run(
-        "sphinx-apidoc",
-        "-o",
-        "api/",
-        "--module-first",
-        "--no-toc",
-        "--force",
-        "../src/jace",
-    )
-
-
 @nox.session
 def build(session: nox.Session) -> None:
     """Build an SDist and wheel."""
-    build_path = DIR.joinpath("build")
+    build_path = ROOT_DIR / "build"
     if build_path.exists():
         shutil.rmtree(build_path)
 
-    session.install("build")
+    session.install(f"build=={REQUIREMENTS['build']}")
     session.run("python", "-m", "build")
diff --git a/pyproject.toml b/pyproject.toml
@@ -24,7 +24,7 @@ classifiers = [
   "Typing :: Typed",
 ]
 dependencies = [
-  "dace>=0.15",
+  "dace>=0.16",
   "jax[cpu]>=0.4.24",
   "numpy>=1.26.0",
 ]
@@ -211,14 +211,20 @@ tests = [
 max-complexity = 12
 
 [tool.ruff.lint.per-file-ignores]
-"!tests/**.py" = ["PT"]  # Ignore flake8-pytest-style outside 'tests/'
+"!tests/**" = ["PT"]  # Ignore flake8-pytest-style outside 'tests/'
 "docs/**" = [
   "D",  # pydocstyle
   "T10",  # flake8-debugger
   "T20",  # flake8-print
 ]
 "noxfile.py" = [
   "D",  # pydocstyle
+  "T10",  # flake8-debugger
+  "T20",  # flake8-print
+]
+"requirements/**" = [
+  "D",  # pydocstyle
+  "T10",  # flake8-debugger
   "T20",  # flake8-print
 ]
 "tests/**" = [

diff --git a/requirements-dev.txt b/requirements-dev.txt
diff --git a/requirements/base.in b/requirements/base.in
@@ -0,0 +1,3 @@
+dace>=0.16
+jax[cpu]>=0.4.24
+numpy>=1.26.0
diff --git a/requirements/base.txt b/requirements/base.txt
@@ -0,0 +1,69 @@
+# SHA1:50585cb1d4e4cc2297a939939d360c886c4ee3e4
+#
+# This file is autogenerated by pip-compile-multi
+# To update, run:
+#
+#    pip-compile-multi
+#
+aenum==3.1.15
+    # via dace
+astunparse==1.6.3
+    # via dace
+dace==0.16
+    # via -r requirements/base.in
+dill==0.3.8
+    # via dace
+fparser==0.1.4
+    # via dace
+jax[cpu]==0.4.29
+    # via -r requirements/base.in
+jaxlib==0.4.29
+    # via jax
+jinja2==3.1.4
+    # via dace
+markupsafe==2.1.5
+    # via jinja2
+ml-dtypes==0.4.0
+    # via
+    #   jax
+    #   jaxlib
+mpmath==1.3.0
+    # via sympy
+networkx==3.3
+    # via dace
+numpy==2.0.0
+    # via
+    #   -r requirements/base.in
+    #   dace
+    #   jax
+    #   jaxlib
+    #   ml-dtypes
+    #   opt-einsum
+    #   scipy
+opt-einsum==3.3.0
+    # via jax
+packaging==24.1
+    # via setuptools-scm
+ply==3.11
+    # via dace
+pyyaml==6.0.1
+    # via dace
+scipy==1.13.1
+    # via
+    #   jax
+    #   jaxlib
+setuptools-scm==8.1.0
+    # via fparser
+six==1.16.0
+    # via astunparse
+sympy==1.12.1
+    # via dace
+tomli==2.0.1
+    # via setuptools-scm
+websockets==12.0
+    # via dace
+wheel==0.43.0
+    # via astunparse
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools