test-and-release.yml

# (C) Copyright 2021 ECMWF.
#
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
# In applying this licence, ECMWF does not waive the privileges and immunities
# granted to it by virtue of its status as an intergovernmental organisation
# nor does it submit to any jurisdiction.
#

name: Tests and release

on:
  workflow_dispatch: {}

  # schedule:
  #   # Every day at 3:07am
  #   # on the "default" branch set in github (currently is develop)
  #   - cron:  '07 3 * * *'

  release:
    types: [created]

  push:
    branches:
      - "*"

  repository_dispatch:
    types:
    - cdsapi-updated
    - magics-python-updated
    - ecmwf-api-client-updated

jobs:
  quality:
    name: Code QA
    runs-on: ubuntu-latest
    steps:
    - run: sudo apt-get install -y pandoc # Needed by sphinx for notebooks
    - uses: actions/checkout@v4
    - uses: actions/setup-python@v5
      with:
        python-version: 3.x
    - uses: pre-commit/action@v3.0.1

  download-test-data:
    name: Download test data
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-python@v4
      with:
        python-version: 3.x

    - name: Tokens
      shell: python
      env:
        ECMWFAPIRC: ${{ secrets.ECMWFAPIRC }}
        CDSAPIRC: ${{ secrets.CDSAPIRC }}

      run: |
        import os
        # for n in ('ECMWFAPIRC', 'CDSAPIRC'):
        n = 'ECMWFAPIRC'
        m = os.path.expanduser("~/." + n.lower())
        if os.environ[n]:
            with open(m, "w") as f:
                print(os.environ[n], file=f)
    - run: mkdir test-data
    - run: echo "dataA" > test-data/dataA.txt
    - run: echo "dataB" > test-data/dataB.txt
    - run: cat test-data/dataA.txt
    - name: Tar files
      run: tar -cvf test-data.tar test-data
    - name: Upload Artifact
      uses: actions/upload-artifact@v3
      with:
        name: test-data
        path: test-data.tar

  tests-with-external-download:
    if: github.event_name == 'release' && github.event.action == 'created'
    strategy:
      fail-fast: true
      matrix:
        platform: ["ubuntu-latest", "macos-latest", "windows-latest"]
        python-version: ["3.10"]
    name: Long tests with external downloads Python ${{ matrix.python-version }} ${{ matrix.platform }}
    runs-on: ${{ matrix.platform }}
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    - name: Tokens
      shell: python
      env:
        ECMWFAPIRC: ${{ secrets.ECMWFAPIRC }}
        CDSAPIRC: ${{ secrets.CDSAPIRC }}
      run: |
        import os
        # for n in ('ECMWFAPIRC', 'CDSAPIRC'):
        n = 'ECMWFAPIRC'
        m = os.path.expanduser("~/." + n.lower())
        if os.environ[n]:
            with open(m, "w") as f:
                print(os.environ[n], file=f)
    - name: Install climetlab
      run: pip install -e .  # run: pip install .[interactive,tensorflow,zarr]
    - run: climetlab versions
    - name: Install test tools
      run: |
        pip install pytest
        pip freeze
    - name: Long Tests
      run: pytest --durations=0 -E release -k 'not test_notebooks'

  tests-notebooks:
    if: github.event_name == 'release' && github.event.action == 'created'
    strategy:
      matrix:
        platform: ["ubuntu-latest"]
        python-version: ["3.9"]
    name: Tests notebooks Python ${{ matrix.python-version }} ${{ matrix.platform }}
    runs-on: ${{ matrix.platform }}
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    - name: Tokens
      shell: python
      env:
        ECMWFAPIRC: ${{ secrets.ECMWFAPIRC }}
        CDSAPIRC: ${{ secrets.CDSAPIRC }}
      run: |
        import os
        # for n in ('ECMWFAPIRC', 'CDSAPIRC'):
        n = 'ECMWFAPIRC'
        m = os.path.expanduser("~/." + n.lower())
        if os.environ[n]:
            with open(m, "w") as f:
                print(os.environ[n], file=f)
    - name: Install climetlab
      run: pip install -e .  # run: pip install .[interactive,tensorflow,zarr]
    - run: climetlab versions
    - name: Install test tools
      run: |
        pip install pytest
        pip freeze
    - name: Tests notebooks
      run: pytest tests/documentation/test_notebooks.py

  short-tests:

    needs: [download-test-data]
    strategy:
      matrix:
        platform: ["ubuntu-latest", "macos-latest", "windows-latest"]
        python-version: ["3.9", "3.10"]

    name: Short tests Python ${{ matrix.python-version }} ${{ matrix.platform }}
    runs-on: ${{ matrix.platform }}
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    - name: Download test-data
      uses: actions/download-artifact@v3
      with:
        name: test-data
    - run: tar xvf test-data.tar
    - run: cat test-data/dataA.txt

    - name: Tokens
      # (this should be removed when tests are refactored)
      shell: python
      env:
        ECMWFAPIRC: ${{ secrets.ECMWFAPIRC }}
        CDSAPIRC: ${{ secrets.CDSAPIRC }}
      run: |
        import os
        # for n in ('ECMWFAPIRC', 'CDSAPIRC'):
        n = 'ECMWFAPIRC'
        m = os.path.expanduser("~/." + n.lower())
        if os.environ[n]:
            with open(m, "w") as f:
                print(os.environ[n], file=f)

    - run: pip install --upgrade pip

    - name: Install climetlab
      run: pip install -e .

    - run: climetlab versions
    - name: Install test tools
      run: |
        pip install pytest
        pip freeze
    - name: Check imports
      run: |
        pytest tests/test_imports.py
      env:
        SKIP_TEST_IMPORTS: 0
    - name: Tests
      run: |
        # -E flag is defined in conftest.py
        # notebooks need more dependencies
        # pytest --durations=0 -E release -k 'not test_notebooks'
        pytest --durations=10 -vv -E short


  deploy:
    name: Upload to Pypi and release
    needs: [short-tests, quality, tests-with-external-download, tests-notebooks]
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.x

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install build wheel twine
      - name: Build and publish
        env:
          TWINE_USERNAME: __token__
          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
        run: |
          python -m build
          twine upload dist/*