Merge pull request #73 from ecmwf/develop

Merge develop into main to prepare release 0.8.1
ecmwf · Nov 1, 2024 · 441bfa4 · 441bfa4
2 parents 594c73a + b522c0f
commit 441bfa4
Show file tree

Hide file tree

Showing 15 changed files with 11,547 additions and 162 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -8,16 +8,25 @@ on:
     - 'develop'
     tags-ignore:
     - '**'
+    paths-ignore:
+    - "docs/**"
+    - "README.md"
 
   # Trigger the workflow on pull request
   pull_request:
+    paths-ignore:
+    - "docs/**"
+    - "README.md"
 
   # Trigger the workflow manually
   workflow_dispatch:
 
   # Trigger after public PR approved for CI
   pull_request_target:
     types: [labeled]
+    paths-ignore:
+    - "docs/**"
+    - "README.md"
 
 jobs:
   # Run CI including downstream packages on self-hosted runners
@@ -28,7 +37,7 @@ jobs:
     with:
       earthkit: ecmwf/earthkit@${{ github.event.pull_request.head.sha || github.sha }}
       codecov_upload: true
-      python_qa: true
+      python_qa: false
     secrets: inherit
 
 

diff --git a/.github/workflows/legacy-ci.yml b/.github/workflows/legacy-ci.yml
@@ -24,43 +24,6 @@ defaults:
     shell: bash -l {0}
 
 jobs:
-  pre-commit:
-    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        ref: ${{ github.event.pull_request.head.sha || github.ref }}
-    - uses: actions/setup-python@v4
-      with:
-        python-version: 3.x
-    - uses: pre-commit/action@v3.0.0
-
-  unit-tests:
-    name: unit-tests (3.10)
-    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        ref: ${{ github.event.pull_request.head.sha || github.ref }}
-    - name: Install Conda environment with Micromamba
-      uses: mamba-org/provision-with-micromamba@v14
-      with:
-        environment-file: tests/environment-unit-tests.yml
-        environment-name: DEVELOP
-        channels: conda-forge
-        cache-env: true
-        extra-specs: |
-          python=3.10
-    - name: Install package
-      run: |
-        python -m pip install . --no-deps
-    - name: Run tests
-      run: |
-        make unit-tests
-
   documentation:
     if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
     runs-on: ubuntu-latest
@@ -73,73 +36,18 @@ jobs:
       with:
         ref: ${{ github.event.pull_request.head.sha || github.ref }}
     - name: Install Conda environment with Micromamba
-      uses: mamba-org/provision-with-micromamba@v12
+      uses: mamba-org/setup-micromamba@v1
       with:
         environment-file: tests/environment-docs-tests.yml
         environment-name: DEVELOP
         channels: conda-forge
-        cache-env: true
-        cache-env-key: ubuntu-latest-3.10
-        extra-specs: |
+        cache-environment: true
+        cache-env-key: ubuntu-latest-3.10-no-eccodes
+        create-args: >-
           python=3.10
     - name: Install package
       run: |
-        python -m pip install --no-deps .
+        python -m pip install --no-deps -e .
     - name: Build documentation
       run: |
         make docs-build
-
-  # integration-tests:
-  #   needs: [unit-tests]
-  #   if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-  #   runs-on: ubuntu-latest
-  #   defaults:
-  #     run:
-  #       shell: bash -l {0}
-
-  #   strategy:
-  #     matrix:
-  #       include:
-  #       - python-version: "3.10"
-  #             # extra: -minver  # This will need to be uncommented and environment-minver.yml updated if we want to publish on conda
-
-  #   steps:
-  #   - uses: actions/checkout@v3
-  #     with:
-  #       ref: ${{ github.event.pull_request.head.sha || github.ref }}
-  #   - name: Install Conda environment with Micromamba
-  #     uses: mamba-org/provision-with-micromamba@v12
-  #     with:
-  #       environment-file: tests/environment-unit-tests${{ matrix.extra }}.yml
-  #       environment-name: DEVELOP${{ matrix.extra }}
-  #       channels: conda-forge
-  #       cache-env: true
-  #       cache-env-key: ubuntu-latest-${{ matrix.python-version }}${{ matrix.extra }}.
-  #       extra-specs: |
-  #         python=${{matrix.python-version }}
-  #   - name: Install package
-  #     run: |
-  #       python -m pip install --no-deps .
-  #   - name: Run tests
-  #     run: |
-  #       make unit-tests
-
-  # distribution:
-  #   needs: [integration-tests, documentation]
-  #   if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-  #   runs-on: ubuntu-latest
-
-  #   steps:
-  #   - uses: actions/checkout@v3
-  #     with:
-  #       ref: ${{ github.event.pull_request.head.sha || github.ref }}
-  #   - name: Build distributions
-  #     run: |
-  #       $CONDA/bin/python -m pip install build
-  #       $CONDA/bin/python -m build
-  #   - name: Publish a Python distribution to PyPI
-  #     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
-  #     uses: pypa/gh-action-pypi-publish@release/v1
-  #     with:
-  #       user: __token__
-  #       password: ${{ secrets.PYPI_API_TOKEN }}
diff --git a/.github/workflows/python-pull-request.yml b/.github/workflows/python-pull-request.yml
@@ -0,0 +1,12 @@
+name: Code Quality checks for PRs
+
+on:
+  push:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  quality:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-precommit-run.yml@v2
+    with:
+      skip-hooks: "no-commit-to-branch"
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,32 +1,48 @@
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.4.0
+  rev: v5.0.0
   hooks:
-  - id: trailing-whitespace
-  - id: end-of-file-fixer
+  - id: trailing-whitespace  # Trailing whitespace checker
+  - id: end-of-file-fixer # Ensure files end in a newline
   - id: check-json
-  - id: check-yaml
+  - id: check-yaml # Check YAML files for syntax errors only
+    args: [--unsafe, --allow-multiple-documents]
   - id: check-toml
   # - id: check-added-large-files
-  - id: debug-statements
+  - id: debug-statements # Check for debugger imports and py37+ breakpoint()
   - id: mixed-line-ending
+  - id: no-commit-to-branch # Prevent committing to main / master
+  - id: check-merge-conflict # Check for files that contain merge conflict
+    exclude: /README\.rst$|^docs/.*\.rst$
 - repo: https://github.com/PyCQA/isort
-  rev: 5.12.0
+  rev: 5.13.2
   hooks:
   - id: isort
+    args:
+    - -l 120
+    - --force-single-line-imports
+    - --profile black
 - repo: https://github.com/psf/black
-  rev: 23.9.1
+  rev: 24.8.0
   hooks:
   - id: black
+    args: [--line-length=120]
 - repo: https://github.com/keewis/blackdoc
   rev: v0.3.8
   hooks:
   - id: blackdoc
     additional_dependencies: [black==23.3.0]
-- repo: https://github.com/PyCQA/flake8
-  rev: 6.1.0
-  hooks:
-  - id: flake8
+    exclude: xr_engine_profile_rst\.py
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.6.9
+  hooks:
+  - id: ruff
+    exclude: '(dev/.*|.*_)\.py$'
+    args:
+    - --line-length=120
+    - --fix
+    - --exit-non-zero-on-fix
+    - --preview
 - repo: https://github.com/executablebooks/mdformat
   rev: 0.7.14
   hooks:
@@ -37,11 +53,11 @@ repos:
   hooks:
   - id: pretty-format-yaml
     args: [--autofix, --preserve-quotes]
-  - id: pretty-format-toml
-    args: [--autofix]
-- repo: https://github.com/PyCQA/pydocstyle.git
-  rev: 6.1.1
-  hooks:
-  - id: pydocstyle
-    additional_dependencies: [toml]
-    exclude: tests|docs
+- repo: https://github.com/sphinx-contrib/sphinx-lint
+  rev: v1.0.0
+  hooks:
+  - id: sphinx-lint
+- repo: https://github.com/tox-dev/pyproject-fmt
+  rev: "2.2.4"
+  hooks:
+  - id: pyproject-fmt
diff --git a/Makefile b/Makefile
@@ -6,14 +6,17 @@ COV_REPORT := html
 setup:
 	pre-commit install
 
-default: qa unit-tests type-check
+default: qa unit-tests
 
 qa:
 	pre-commit run --all-files
 
 unit-tests:
 	cd .. && python -m pytest earthkit -vv --cov=. --cov-report=$(COV_REPORT) && cd -
 
+type-check:
+	python -m mypy .
+
 conda-env-update:
 	$(CONDA) env update $(CONDAFLAGS) -f environment.yml
 

diff --git a/docs/examples/01_grib_file.ipynb b/docs/examples/01_grib_file.ipynb
@@ -44,7 +44,6 @@
    "source": [
     "import io\n",
     "import matplotlib.pyplot as plt\n",
-    "import numpy as np\n",
     "from earthkit import data, regrid\n",
     "\n",
     "fs = data.from_source(\"file\", \"tz_1000.grib\")"

diff --git a/docs/examples/02_grib_fdb.ipynb b/docs/examples/02_grib_fdb.ipynb
@@ -35,7 +35,6 @@
    "source": [
     "import io\n",
     "import matplotlib.pyplot as plt\n",
-    "import numpy as np\n",
     "import pyfdb\n",
     "from earthkit import data, regrid"
    ]