Task/geospatial tests #10300
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| jobs: | |
| lint-api: | |
| name: Python - Lint | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| matrix: | |
| python-version: [3.10.4] | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| - name: Install ubuntu pre-requisites (api) | |
| # The python gdal and R component relies on libgdal-dev being installed. | |
| # cffdrs requires libudunits2-dev | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get -y install libgdal-dev libudunits2-dev | |
| - name: Setup Python ${{ matrix.python-version }} (api) | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Cache poetry installer | |
| uses: actions/cache@v4 | |
| id: cache-poetry-installer | |
| env: | |
| cache-name: cache-poetry-installer | |
| with: | |
| path: "~/poetry_installer" | |
| key: "poetry-installer-1.6.1" | |
| - name: Download poetry installer | |
| if: steps.cache-poetry-installer.outputs.cache-hit != 'true' | |
| run: | | |
| echo | |
| mkdir ~/poetry_installer | |
| curl -sSL https://install.python-poetry.org > ~/poetry_installer/install-poetry.py | |
| - name: Install poetry (api) | |
| run: | | |
| cd ~/poetry_installer | |
| python install-poetry.py --version 1.6.1 | |
| poetry config virtualenvs.create true | |
| poetry config virtualenvs.in-project false | |
| - name: Cache poetry | |
| uses: actions/cache@v4 | |
| env: | |
| cache-name: cache-poetry | |
| with: | |
| path: ~/.cache/pypoetry | |
| key: ${{ runner.os }}-poetry-1.6.1-cache-${{ hashFiles('**/poetry.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-poetry-1.6.1-cache- | |
| - name: Cache .venv | |
| id: cache-venv | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/work/wps/wps/api/.venv | |
| key: ${{ runner.os }}-venv-poetry-1.6.1-${{ hashFiles('**/poetry.lock') }} | |
| - name: Install python dependencies using poetry (api) | |
| if: steps.cache-venv.outputs.cache-hit != 'true' | |
| working-directory: ./api | |
| run: | | |
| poetry config http-basic.psu ${{ secrets.ARTIFACTORY_PYPI_USERNAME }} ${{ secrets.ARTIFACTORY_PYPI_PASSWORD }} | |
| poetry run python -m pip install --upgrade pip | |
| poetry install | |
| poetry run python -m pip install gdal==$(gdal-config --version) | |
| - name: Lint (api) | |
| # We used to be able to do linting before installing gdal, but it's not possible anymore. | |
| # We can however place it ahead of the R installs. | |
| working-directory: ./api | |
| run: | | |
| poetry run ruff app/*.py app/**/*.py | |
| test-api: | |
| name: Python - Test with coverage | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| matrix: | |
| # Match versions for python + R to Ubuntu 22.04 LTS at the time of writing. | |
| python-version: [3.10.4] | |
| R: ["4.1.2"] | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| with: | |
| # For sonar-scanner to work properly we can't use a shallow fetch. | |
| fetch-depth: 0 | |
| - name: Install ubuntu pre-requisites (api) | |
| # The python gdal and R component relies on libgdal-dev being installed. | |
| # cffdrs requires libudunits2-dev | |
| # The api uses wkhtmltopdf to generate pdf's. | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get -y install libgdal-dev wkhtmltopdf libudunits2-dev | |
| - name: Setup Python ${{ matrix.python-version }} (api) | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Cache poetry installer | |
| uses: actions/cache@v4 | |
| id: cache-poetry-installer | |
| env: | |
| cache-name: cache-poetry-installer | |
| with: | |
| path: "~/poetry_installer" | |
| key: "poetry-installer-1.6.1" | |
| - name: Download poetry installer | |
| if: steps.cache-poetry-installer.outputs.cache-hit != 'true' | |
| run: | | |
| echo | |
| mkdir ~/poetry_installer | |
| curl -sSL https://install.python-poetry.org > ~/poetry_installer/install-poetry.py | |
| - name: Install poetry (api) | |
| run: | | |
| cd ~/poetry_installer | |
| python install-poetry.py --version 1.6.1 | |
| poetry config virtualenvs.create true | |
| poetry config virtualenvs.in-project false | |
| # poetry cache folder: /home/runner/.cache/pypoetry | |
| - name: Cache poetry | |
| uses: actions/cache@v4 | |
| env: | |
| cache-name: cache-poetry | |
| with: | |
| path: ~/.cache/pypoetry | |
| key: ${{ runner.os }}-poetry-1.6.1-cache-${{ hashFiles('**/poetry.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-poetry-1.6.1-cache- | |
| - name: Cache .venv | |
| id: cache-venv | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/work/wps/wps/api/.venv | |
| key: ${{ runner.os }}-venv-poetry-1.6.1-${{ hashFiles('**/poetry.lock') }} | |
| - name: Install python dependencies using poetry (api) | |
| if: steps.cache-venv.outputs.cache-hit != 'true' | |
| working-directory: ./api | |
| run: | | |
| poetry config http-basic.psu ${{ secrets.ARTIFACTORY_PYPI_USERNAME }} ${{ secrets.ARTIFACTORY_PYPI_PASSWORD }} | |
| poetry run python -m pip install --upgrade pip | |
| poetry install | |
| poetry run python -m pip install gdal==$(gdal-config --version) | |
| - uses: r-lib/actions/setup-r@v2 | |
| # r-lib/actions/setup-r@v2 is supposed to install version ${{ matrix.R }}, BUT, it doesn't. | |
| # When asking for 4.1.2 - it's installing a new version! Or at least reporting a newer | |
| # version. | |
| with: | |
| r-version: ${{ matrix.R }} | |
| - name: Cache /home/runner/work/_temp/Library | |
| # When installing cffdrs, a bunch of stuff is downloaded and compiled and placed in | |
| # /home/runner/work/_temp/Library ; By caching this folder, subsequent calls to | |
| # install cffdrs run much faster. | |
| id: cache-r-cffdrs | |
| uses: actions/cache@v4 | |
| with: | |
| path: /home/runner/work/_temp/Library | |
| key: ${{ runner.os }}-r-${{ matrix.R }}-cffdrs | |
| - name: Install R dependencies (api) | |
| run: R -e "install.packages('cffdrs')" | |
| - name: Unit Test with coverage (api) | |
| working-directory: ./api | |
| run: | | |
| export LD_LIBRARY_PATH=$(poetry run python -m rpy2.situation LD_LIBRARY_PATH):${LD_LIBRARY_PATH} | |
| export CLASSPATH=./libs/REDapp_Lib.jar:./libs/WTime.jar:./libs/hss-java.jar:$CLASSPATH | |
| export ORIGINS=testorigin | |
| export SFMS_SECRET=secret | |
| poetry run coverage run --source=app -m pytest app/tests -x -o log_cli=true --disable-warnings -vvv | |
| - name: Create coverage report (api) | |
| working-directory: ./api | |
| shell: bash | |
| run: | | |
| poetry run coverage report | |
| poetry run coverage xml -o coverage-reports/coverage-report.xml | |
| - name: Archive coverage report (api) | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: api-coverage-report | |
| path: ./api/coverage-reports/coverage-report.xml | |
| lint-and-test-web: | |
| name: Web - Lint, Test with coverage | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| matrix: | |
| node-version: [20.x] | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| with: | |
| # For sonar-scanner to work properly we can't use a shallow fetch. | |
| fetch-depth: 0 | |
| - name: Setup kernel for react, increase watchers | |
| run: echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p | |
| - name: Use Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| **/node_modules | |
| ~/.cache/Cypress | |
| key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }} | |
| - name: enable corepack | |
| run: corepack enable | |
| - name: use new yarn | |
| run: yarn set version berry | |
| - name: Setup yarn for scoped artifactory packages | |
| working-directory: ./web | |
| run: yarn config set npmScopes.psu.npmRegistryServer https://artifacts.developer.gov.bc.ca/artifactory/api/npm/pe1e-psu-npm-local/ && yarn config set npmScopes.psu.npmAlwaysAuth true && yarn config set npmScopes.psu.npmAuthToken ${{ secrets.ARTIFACTORY_SVCACCT_TOKEN }} | |
| - name: Install node dependencies (web) | |
| working-directory: ./web | |
| if: steps.yarn-cache.outputs.cache-hit != 'true' | |
| run: yarn install | |
| - name: Lint (web) | |
| working-directory: ./web | |
| run: yarn run lint | |
| # "Error: ENOSPC: System limit for number of file watchers reached" can be addressed | |
| # with this: https://github.com/guard/listen/wiki/Increasing-the-amount-of-inotify-watchers#the-technical-details | |
| # It seems unnecessary at the moment because tests pass anyway | |
| - name: Cypress tests with coverage (web) | |
| working-directory: ./web | |
| run: yarn run cypress:ci | |
| - name: Unit tests (web) | |
| working-directory: ./web | |
| run: yarn run coverage:ci | |
| - name: Merge and finalize test coverage (web) | |
| working-directory: ./web | |
| run: yarn run finalizeCoverage | |
| - name: Archive coverage report | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: web-coverage-report | |
| path: ./web/finalCoverage | |
| upload-code-coverage: | |
| name: Coverage with CodeCov | |
| runs-on: ubuntu-22.04 | |
| needs: [lint-and-test-web, test-api] | |
| steps: | |
| # we need to checkout, so that we have codecov.yml | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| - name: Download all workflow run artifacts | |
| uses: actions/download-artifact@v3 | |
| - name: Upload test coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| token: ${{secrets.CODECOV_TOKEN}} | |
| # we want to ensure code coverage is good, so we fail on error. (default is false) | |
| fail_ci_if_error: true | |
| lint-and-test-prune: | |
| name: Backup Prune - Lint and Test | |
| # We have to use an older ubuntu version, because of the older version of python that the | |
| # backup script uses. | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| matrix: | |
| python-version: [3.11.2] | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| with: | |
| # For sonar-scanner to work properly we can't use a shallow fetch. | |
| fetch-depth: 0 | |
| - name: Setup Python ${{ matrix.python-version }} (api) | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Cache poetry installer | |
| uses: actions/cache@v4 | |
| id: cache-poetry-installer | |
| env: | |
| cache-name: cache-poetry-installer | |
| with: | |
| path: "~/poetry_installer" | |
| key: "poetry-installer-1.7.1" | |
| - name: Download poetry installer | |
| if: steps.cache-poetry-installer.outputs.cache-hit != 'true' | |
| run: | | |
| echo | |
| mkdir ~/poetry_installer | |
| curl -sSL https://install.python-poetry.org > ~/poetry_installer/install-poetry.py | |
| - name: Install poetry (api) | |
| run: | | |
| cd ~/poetry_installer | |
| python install-poetry.py --version 1.7.1 | |
| poetry config virtualenvs.create true | |
| poetry config virtualenvs.in-project false | |
| # poetry cache folder: /home/runner/.cache/pypoetry | |
| - name: Cache poetry | |
| uses: actions/cache@v4 | |
| env: | |
| cache-name: cache-poetry | |
| with: | |
| path: ~/.cache/pypoetry | |
| key: ${{ runner.os }}-poetry-prune-cache-${{ hashFiles('**/poetry.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-poetry-prune-cache- | |
| - name: Install python dependencies using poetry (s3-backup) | |
| working-directory: ./openshift/s3-backup/docker | |
| run: | | |
| poetry install | |
| - name: Lint (s3-backup) | |
| working-directory: ./openshift/s3-backup/docker | |
| run: poetry run pylint *.py | |
| - name: Unit tests (s3-backup) | |
| working-directory: ./openshift/s3-backup/docker | |
| run: | | |
| poetry run pytest -v |