Skip to content
Integration

ASA Map Legend #8347

Sign in to view logs

ASA Map Legend

ASA Map Legend #8347

Workflow file for this run

.github/workflows/integration.yml at 130cb40
name: Integration
on:
pull_request:
branches:
- main
jobs:
lint-api:
name: Python - Lint
runs-on: ubuntu-22.04
strategy:
matrix:
python-version: [3.10.4]
steps:
- name: Checkout repo
uses: actions/checkout@v4
- name: Install ubuntu pre-requisites (api)
# The python gdal and R component relies on libgdal-dev being installed.
run: |
sudo apt-get update
sudo apt-get -y install libgdal-dev
- name: Setup Python ${{ matrix.python-version }} (api)
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Cache poetry installer
uses: actions/cache@v3
id: cache-poetry-installer
env:
cache-name: cache-poetry-installer
with:
path: "~/poetry_installer"
key: "poetry-installer-1.3.1"
- name: Download poetry installer
if: steps.cache-poetry-installer.outputs.cache-hit != 'true'
run: |
echo
mkdir ~/poetry_installer
curl -sSL https://install.python-poetry.org > ~/poetry_installer/install-poetry.py
- name: Install poetry (api)
run: |
cd ~/poetry_installer
python install-poetry.py --version 1.3.1
poetry config virtualenvs.create true
poetry config virtualenvs.in-project false
- name: Cache poetry
uses: actions/cache@v3
env:
cache-name: cache-poetry
with:
path: ~/.cache/pypoetry
key: ${{ runner.os }}-poetry-1.3.1-cache-${{ hashFiles('**/poetry.lock') }}
restore-keys: |
${{ runner.os }}-poetry-1.3.1-cache-
- name: Cache .venv
id: cache-venv
uses: actions/cache@v3
with:
path: ~/work/wps/wps/api/.venv
key: ${{ runner.os }}-venv-poetry-1.3.1-${{ hashFiles('**/poetry.lock') }}
- name: Install python dependencies using poetry (api)
if: steps.cache-venv.outputs.cache-hit != 'true'
working-directory: ./api
run: |
poetry run python -m pip install --upgrade pip
poetry install
poetry run python -m pip install gdal==$(gdal-config --version)
- name: Lint (api)
# We used to be able to do linting before installing gdal, but it's not possible anymore.
# We can however place it ahead of the R installs.
working-directory: ./api
run: |
poetry run ruff app/*.py app/**/*.py
test-api:
name: Python - Test with coverage
runs-on: ubuntu-22.04
strategy:
matrix:
# Match versions for python + R to Ubuntu 22.04 LTS at the time of writing.
python-version: [3.10.4]
R: ["4.1.2"]
steps:
- name: Checkout repo
uses: actions/checkout@v4
with:
# For sonar-scanner to work properly we can't use a shallow fetch.
fetch-depth: 0
- name: Install ubuntu pre-requisites (api)
# The python gdal and R component relies on libgdal-dev being installed.
# The api uses wkhtmltopdf to generate pdf's.
run: |
sudo apt-get update
sudo apt-get -y install libgdal-dev wkhtmltopdf
- name: Setup Python ${{ matrix.python-version }} (api)
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Cache poetry installer
uses: actions/cache@v3
id: cache-poetry-installer
env:
cache-name: cache-poetry-installer
with:
path: "~/poetry_installer"
key: "poetry-installer-1.3.1"
- name: Download poetry installer
if: steps.cache-poetry-installer.outputs.cache-hit != 'true'
run: |
echo
mkdir ~/poetry_installer
curl -sSL https://install.python-poetry.org > ~/poetry_installer/install-poetry.py
- name: Install poetry (api)
run: |
cd ~/poetry_installer
python install-poetry.py --version 1.3.1
poetry config virtualenvs.create true
poetry config virtualenvs.in-project false
# poetry cache folder: /home/runner/.cache/pypoetry
- name: Cache poetry
uses: actions/cache@v3
env:
cache-name: cache-poetry
with:
path: ~/.cache/pypoetry
key: ${{ runner.os }}-poetry-1.3.1-cache-${{ hashFiles('**/poetry.lock') }}
restore-keys: |
${{ runner.os }}-poetry-1.3.1-cache-
- name: Cache .venv
id: cache-venv
uses: actions/cache@v3
with:
path: ~/work/wps/wps/api/.venv
key: ${{ runner.os }}-venv-poetry-1.3.1-${{ hashFiles('**/poetry.lock') }}
- name: Install python dependencies using poetry (api)
if: steps.cache-venv.outputs.cache-hit != 'true'
working-directory: ./api
run: |
poetry run python -m pip install --upgrade pip
poetry install
poetry run python -m pip install gdal==$(gdal-config --version)
- uses: r-lib/actions/setup-r@v2
# r-lib/actions/setup-r@v2 is supposed to install version ${{ matrix.R }}, BUT, it doesn't.
# When asking for 4.1.2 - it's installing a new version! Or at least reporting a newer
# version.
with:
r-version: ${{ matrix.R }}
- name: Cache /home/runner/work/_temp/Library
# When installing cffdrs, a bunch of stuff is downloaded and compiled and placed in
# /home/runner/work/_temp/Library ; By caching this folder, subsequent calls to
# install cffdrs run much faster.
id: cache-r-cffdrs
uses: actions/cache@v3
with:
path: /home/runner/work/_temp/Library
key: ${{ runner.os }}-r-${{ matrix.R }}-cffdrs
- name: Install R dependencies (api)
run: R -e "install.packages('cffdrs')"
- name: Unit Test with coverage (api)
working-directory: ./api
run: |
export LD_LIBRARY_PATH=$(poetry run python -m rpy2.situation LD_LIBRARY_PATH):${LD_LIBRARY_PATH}
export CLASSPATH=./libs/REDapp_Lib.jar:./libs/WTime.jar:./libs/hss-java.jar:$CLASSPATH
export ORIGINS=testorigin
export SFMS_SECRET=secret
poetry run coverage run --source=app -m pytest app/tests -x -o log_cli=true --disable-warnings -vvv
- name: Create coverage report (api)
working-directory: ./api
shell: bash
run: |
poetry run coverage report
poetry run coverage xml -o coverage-reports/coverage-report.xml
- name: Archive coverage report (api)
uses: actions/upload-artifact@v3
with:
name: api-coverage-report
path: ./api/coverage-reports/coverage-report.xml
lint-and-test-web:
name: Web - Lint, Test with coverage
runs-on: ubuntu-22.04
strategy:
matrix:
node-version: [16.x]
steps:
- name: Checkout repo
uses: actions/checkout@v4
with:
# For sonar-scanner to work properly we can't use a shallow fetch.
fetch-depth: 0
- name: Setup kernel for react, increase watchers
run: echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
- uses: actions/cache@v3
with:
path: |
**/node_modules
~/.cache/Cypress
key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
- name: Install node dependencies (web)
working-directory: ./web
if: steps.yarn-cache.outputs.cache-hit != 'true'
run: yarn install
- name: Lint (web)
working-directory: ./web
run: yarn run lint
# "Error: ENOSPC: System limit for number of file watchers reached" can be addressed
# with this: https://github.com/guard/listen/wiki/Increasing-the-amount-of-inotify-watchers#the-technical-details
# It seems unnecessary at the moment because tests pass anyway
- name: Cypress tests with coverage (web)
working-directory: ./web
run: yarn run cypress:ci
- name: Unit tests (web)
working-directory: ./web
run: yarn run coverage:ci
- name: Merge and finalize test coverage (web)
working-directory: ./web
run: yarn run finalizeCoverage
- name: Archive coverage report
uses: actions/upload-artifact@v3
with:
name: web-coverage-report
path: ./web/finalCoverage
upload-code-coverage:
name: Coverage with CodeCov
runs-on: ubuntu-22.04
needs: [lint-and-test-web, test-api]
steps:
# we need to checkout, so that we have codecov.yml
- name: Checkout repo
uses: actions/checkout@v4
- name: Download all workflow run artifacts
uses: actions/download-artifact@v3
- name: Upload test coverage to Codecov
uses: codecov/codecov-action@v3
with:
token: ${{secrets.CODECOV_TOKEN}}
# we want to ensure code coverage is good, so we fail on error. (default is false)
fail_ci_if_error: true
lint-and-test-prune:
name: Backup Prune - Lint and Test
# We have to use an older ubuntu version, because of the older version of python that the
# backup script uses.
runs-on: ubuntu-20.04
strategy:
matrix:
python-version: [3.6.15]
steps:
- name: Checkout repo
uses: actions/checkout@v4
with:
# For sonar-scanner to work properly we can't use a shallow fetch.
fetch-depth: 0
- name: Setup Python ${{ matrix.python-version }} (api)
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Cache poetry installer
uses: actions/cache@v3
id: cache-poetry-installer
env:
cache-name: cache-poetry-installer
with:
path: "~/poetry_installer"
key: "poetry-installer-1.1.11"
- name: Download poetry installer
if: steps.cache-poetry-installer.outputs.cache-hit != 'true'
run: |
echo
mkdir ~/poetry_installer
curl -sSL https://install.python-poetry.org > ~/poetry_installer/install-poetry.py
- name: Install poetry (api)
run: |
cd ~/poetry_installer
python install-poetry.py --version 1.1.11
poetry config virtualenvs.create true
poetry config virtualenvs.in-project false
poetry config experimental.new-installer false
# poetry cache folder: /home/runner/.cache/pypoetry
- name: Cache poetry
uses: actions/cache@v3
env:
cache-name: cache-poetry
with:
path: ~/.cache/pypoetry
key: ${{ runner.os }}-poetry-prune-cache-${{ hashFiles('**/poetry.lock') }}
restore-keys: |
${{ runner.os }}-poetry-prune-cache-
- name: Install python dependencies using poetry (s3-backup)
working-directory: ./openshift/s3-backup/docker
run: |
poetry install
- name: Lint (s3-backup)
working-directory: ./openshift/s3-backup/docker
run: poetry run pylint *.py
- name: Unit tests (s3-backup)
working-directory: ./openshift/s3-backup/docker
run: |
poetry run pytest -v