Adding first gis module to perform geospatial operations, notebook builds

.github/labeler.yml

@@ -18,3 +18,8 @@
       - '.github/workflows/*'
       - 'tox.ini'
       - 'Makefile'
+
+'notebooks':
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'docs/notebooks/**/*.ipynb'

.github/workflows/label-on-approval.yml

@@ -0,0 +1,76 @@
+name: Label Pull Request on Pull Request review approval
+
+on:
+  pull_request_review:
+    types:
+      - submitted
+  pull_request_target:
+    types:
+      - ready_for_review
+      - review_requested
+
+permissions:
+  checks: write
+  contents: read
+  pull-requests: write
+
+jobs:
+  label_approved:
+    name: Label on Approval
+    if: |
+      (!contains(github.event.pull_request.labels.*.name, 'approved')) &&
+      (github.event.review.state == 'approved') &&
+      (github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Label Approved
+        uses: actions/github-script@v7.0.1
+        with:
+          script: |
+            github.rest.issues.addLabels({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              labels: ['approved']
+            })
+
+  comment_approved:
+    name: Comment Concerning Approved Tag
+    if: |
+      (github.event_name == 'pull_request_target') &&
+      (github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Find comment
+        uses: peter-evans/find-comment@v2.4.0
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: This Pull Request is coming from a fork and must be manually tagged `approved` in order to perform additional testing
+      - name: Create comment
+        if: |
+          (steps.fc.outputs.comment-id == '') &&
+          (!contains(github.event.pull_request.labels.*.name, 'approved')) &&
+          (github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)
+        uses: peter-evans/create-or-update-comment@v3.1.0
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body: |
+            > **Warning**
+            > This Pull Request is coming from a fork and must be manually tagged `approved` in order to perform additional testing.
+          edit-mode: replace
+      - name: Update comment
+        if: |
+          contains(github.event.pull_request.labels.*.name, 'approved')
+        uses: peter-evans/create-or-update-comment@v3.1.0
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body: |
+            > **Note**
+            > This Pull Request is approved!
+          reactions: |
+            hooray
+          edit-mode: append

.github/workflows/main.yml

@@ -12,6 +12,13 @@ on:
       - tests/test_xhydro.py
       - xhydro/__init__.py
   pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+  pull_request_review:
+    types:
+      - submitted
 
 concurrency:
   # For a given workflow, if we push to the same branch, cancel all previous builds on that branch except on master.
@@ -25,6 +32,10 @@ jobs:
   lint:
     name: Lint (Python${{ matrix.python-version }})
     runs-on: ubuntu-latest
+    if: |
+      ((github.event_name == 'pull_request') && (github.event.action != 'labeled')) ||
+      (github.event.review.state == 'approved') ||
+      (github.event_name == 'push')
     strategy:
       matrix:
         python-version:
@@ -42,9 +53,54 @@ jobs:
         run: |
           python -m tox -e lint
 
+  test-preliminary:
+    name: Preliminary Tests (Python${{ matrix.python-version }})
+    needs: lint
+    if: |
+      (github.event_name == 'pull_request') && !contains(github.event.pull_request.labels.*.name, 'approved')
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - python-version: "3.9"
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Conda (Micromamba) with Python${{ matrix.python-version }}
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          cache-downloads: true
+          environment-file: environment-dev.yml
+          create-args: >-
+            mamba
+            python=${{ matrix.python-version }}
+      - name: Conda and Mamba versions
+        run: |
+          mamba --version
+          echo "micromamba $(micromamba --version)"
+      - name: Install xHydro
+        run: |
+          python -m pip install --no-deps .
+      - name: Check versions
+        run: |
+          conda list
+          echo ESMF_VERSION=$(cat $ESMFMKFILE | grep "ESMF_VERSION_STRING=" | awk -F= '{print $2}' | tr -d "'")
+          python -m pip check || true
+      - name: Test with pytest
+        run: |
+          make test
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
   test-conda:
     name: Test with Python${{ matrix.python-version }} (Anaconda)
     needs: lint
+    if: |
+      contains(github.event.pull_request.labels.*.name, 'approved') ||
+      (github.event.review.state == 'approved') ||
+      (github.event_name == 'push')
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -81,6 +137,7 @@ jobs:
       - name: Check versions
         run: |
           conda list
+          echo ESMF_VERSION=$(cat $ESMFMKFILE | grep "ESMF_VERSION_STRING=" | awk -F= '{print $2}' | tr -d "'")
           python -m pip check || true
       - name: Test with pytest
         run: |
@@ -94,9 +151,99 @@ jobs:
           COVERALLS_PARALLEL: true
           COVERALLS_SERVICE_NAME: github
 
+  test-notebooks:
+    name: Test Notebooks
+    needs: lint
+    if: |
+      contains(github.event.pull_request.labels.*.name, 'notebooks') ||
+      contains(github.event.pull_request.labels.*.name, 'approved') ||
+      (github.event.review.state == 'approved') ||
+      (github.event_name == 'push')
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+       python-version: ["3.9", "3.10", "3.11", "3.12"]
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Conda (Micromamba) with Python${{ matrix.python-version }}
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          cache-downloads: true
+          environment-file: environment-dev.yml
+          create-args: >-
+            mamba
+            python=${{ matrix.python-version }}
+      - name: Conda and Mamba versions
+        run: |
+          mamba --version
+          echo "micromamba $(micromamba --version)"
+      - name: Install xHydro
+        run: |
+          python -m pip install --no-deps .
+      - name: Check versions
+        run: |
+          conda list
+          echo ESMF_VERSION=$(cat $ESMFMKFILE | grep "ESMF_VERSION_STRING=" | awk -F= '{print $2}' | tr -d "'")
+          python -m pip check || true
+      - name: Test Notebooks
+        run: |
+          make test-notebooks-lax
+
+#  test-ESMF-source:
+#    name: Tests using ESMF from sources (Python${{ matrix.python-version }})
+#    needs: lint
+#    if: |
+#      contains(github.event.pull_request.labels.*.name, 'approved') ||
+#      (github.event.review.state == 'approved') ||
+#      (github.event_name == 'push')
+#    runs-on: ubuntu-latest
+#    env:
+#      ESMF_VERSION: "v8.5.0"
+#    strategy:
+#      matrix:
+#        include:
+#          - python-version: "3.9"
+#    steps:
+#      - uses: actions/checkout@v4.1.1
+#      - name: Install NetCDF
+#        run: |
+#          sudo apt-get -y update
+#          sudo apt install libnetcdf-dev libnetcdff-dev
+#      - name: Install ESMF
+#        uses: esmf-org/install-esmf-action@v1
+#        env:
+#          ESMF_NETCDF: nc-config
+#        with:
+#          cache: true
+#          version: ${{ env.ESMF_VERSION }}
+#      - name: Set up Python${{ matrix.python-version }}
+#        uses: actions/setup-python@v4.7.1
+#        with:
+#          python-version: ${{ matrix.python-version }}
+#      - name: Install xhydro (with esmpy)
+#        run: |
+#          python -m pip install flit
+#          make dev
+#        env:
+#          ESMF_VERSION: ${{ env.ESMF_VERSION }}
+#      - name: Check versions
+#        run: |
+#          python -m pip list
+#          echo ESMF_VERSION=$(cat $ESMFMKFILE | grep "ESMF_VERSION_STRING=" | awk -F= '{print $2}' | tr -d "'")
+#          python -m pip check || true
+#      - name: Test with pytest
+#        run: |
+#          make test
+#        env:
+#          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
   finish:
     needs:
       - test-conda
+      - test-notebooks
     runs-on: ubuntu-latest
     container: python:3-slim
     steps:

.pre-commit-config.yaml

@@ -17,7 +17,7 @@ repos:
       - id: check-toml
       - id: pretty-format-json
         args: [ '--autofix', '--no-ensure-ascii', '--no-sort-keys' ]
-        # exclude: .ipynb
+        exclude: .ipynb
       - id: check-yaml
         args: [ '--allow-multiple-documents' ]
   - repo: https://github.com/pappasam/toml-sort
@@ -61,6 +61,22 @@ repos:
     hooks:
       - id: blackdoc
         additional_dependencies: [ 'black==24.2.0' ]
+  - repo: https://github.com/kynan/nbstripout
+    rev: 0.6.1
+    hooks:
+      - id: nbstripout
+        files: '.ipynb'
+        args: [ '--keep-count', '--keep-output', '--drop-empty-cells', '--extra-keys=metadata.kernelspec' ]
+  - repo: https://github.com/nbQA-dev/nbQA
+    rev: 1.7.1
+    hooks:
+      - id: nbqa-pyupgrade
+        args: [ '--py39-plus' ]
+        additional_dependencies: [ 'pyupgrade==3.15.0' ]
+      - id: nbqa-black
+        additional_dependencies: [ 'black==24.2.0' ]
+      - id: nbqa-isort
+        additional_dependencies: [ 'isort==5.13.2' ]
   - repo: https://github.com/adrienverge/yamllint.git
     rev: v1.35.1
     hooks:

CHANGES.rst

@@ -2,6 +2,26 @@
 Changelog
 =========
 
+v0.4.0 (unreleased)
+-------------------
+Contributors to this version: Trevor James Smith (:user:`Zeitsperre`), Thomas-Charles Fortier Filion (:user:`TC-FF`), Sébastien Langlois (:user:`sebastienlanglois`).
+
+New features and enhancements
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+* `xhydro` has implemented a `gis` module that facilitates geospatial tasks needed for gathering hydrological inputs. (:issue:`60`, :pull:`61`).
+
+Internal changes
+^^^^^^^^^^^^^^^^
+* Added a workflow based on `actions/labeler` to automatically label Pull Requests based on files changed. (:pull:`68`).
+* Added a conditional trigger to the `test-notebooks` job to run in advance of pull request approval in the event that the notebooks found within `docs/notebooks` have been modified (labeled `"notebooks"`). (:pull:`68`).
+* Significant changes to the Continuous Integration (CI) setup. (:pull:`65`):
+    * Added a workflow configuration using ``label_on_approval.yml`` and modifications of ``main.yml`` so that fewer tests are run on Pull Requests before they are fully approved.
+    * Added some `pre-commit` configurations to both clean up the code within notebooks (`NbQA`) and strip their outputs (`nbstripout`).
+    * `tox` is now fully v4.0-compliant.
+    * Added a `Makefile` recipe to facilitate installation of `esmpy` when `esmf` is installed and visible on the `$PATH`.
+    * Added a `Makefile` recipe for running tests over Jupyter notebooks.
+    * Synchronized dependencies between `pyproject.toml` and `conda` configuration files.
+
 v0.3.4 (2024-02-29)
 -------------------
 Contributors to this version: Trevor James Smith (:user:`Zeitsperre`), Thomas-Charles Fortier Filion (:user:`TC-FF`), Gabriel Rondeau-Genesse (:user:`RondeauG`).

Makefile

@@ -68,6 +68,12 @@ lint: lint/flake8 lint/black ## check style
 test: ## run tests quickly with the default Python
 	python -m pytest
 
+test-notebooks: ## run tests on notebooks and compare outputs
+	pytest --no-cov --nbval --rootdir=tests/ docs/notebooks
+
+test-notebooks-lax: ## run tests on notebooks but don't be so strict about outputs
+	pytest --no-cov --nbval-lax --rootdir=tests/ docs/notebooks
+
 test-all: ## run tests on every Python version with tox
 	tox
 
@@ -107,8 +113,12 @@ dist: clean ## builds source and wheel package
 release: dist ## package and upload a release
 	python -m flit publish dist/*
 
-install: clean ## install the package to the active Python's site-packages
+ESMF_VERSION := $(shell cat $(ESMFMKFILE) | grep "ESMF_VERSION_STRING=" | awk -F= '{print $$2}' | tr -d "'")
+install-esmpy: clean ## install esmpy from git based on installed ESMF_VERSION
+	pip install git+https://github.com/esmf-org/esmf.git@v$(ESMF_VERSION)\#subdirectory=src/addon/esmpy
+
+install: install-esmpy ## install the package to the active Python's site-packages
 	python -m flit install
 
-dev: clean ## install the package to the active Python's site-packages
+dev: install-esmpy ## install the package to the active Python's site-packages
 	python -m flit install --symlink

docs/conf.py

@@ -16,6 +16,7 @@
 # directory, add these directories to sys.path here. If the directory is
 # relative to the documentation root, use os.path.abspath to make it
 # absolute, like shown here.
+import warnings
 from pathlib import Path
 
 import os
@@ -77,7 +78,14 @@
 }
 
 # Generate documentation from Jupyter notebooks.
-nbsphinx_execute = "always"
+skip_notebooks = os.getenv("SKIP_NOTEBOOKS")
+if skip_notebooks:
+    warnings.warn("Not executing notebooks.")
+    nbsphinx_execute = "never"
+elif os.getenv("GITHUB_ACTIONS"):
+    nbsphinx_execute = "always"
+else:
+    nbsphinx_execute = "auto"
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]

docs/index.rst

@@ -9,6 +9,7 @@ Welcome to xHydro's documentation!
    readme
    installation
    usage
+   notebooks/gis
    notebooks/local_frequency_analysis
    notebooks/climate_change
    planification