release v2.18 #468
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# For more information see: | |
# https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions | |
name: OPERANDI CI/CD | |
on: | |
push: | |
# branches: [ main ] | |
paths: | |
- .github/** | |
- src/** | |
- tests/** | |
- .* | |
- docker* | |
- Makefile | |
pull_request: | |
# branches: [ main ] | |
paths: | |
- .github/** | |
- src/** | |
- tests/** | |
- .* | |
- docker* | |
- Makefile | |
env: | |
REGISTRY: ghcr.io | |
REPO_NAME: ${{ github.repository }} | |
jobs: | |
build-native: | |
name: Native build of Operandi modules | |
strategy: | |
fail-fast: true | |
matrix: | |
python-version: ["3.8", "3.9", "3.10", "3.11"] | |
os: [ubuntu-latest] | |
runs-on: ${{ matrix.os }} | |
steps: | |
- name: Checkout repository | |
uses: actions/checkout@v3 | |
with: | |
ref: ${{ github.head_ref }} | |
- name: Set up Python ${{ matrix.python-version }} | |
uses: actions/setup-python@v4 | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: Install operandi dependencies | |
run: | | |
sudo apt-get update && sudo apt-get -y install make | |
python3 -m pip install --upgrade pip setuptools | |
pip3 install -U pip wheel | |
pip install -r tests/requirements.txt | |
- name: Lint with flake8 | |
run: | | |
python3 -m pip install flake8 | |
# stop the build if there are Python syntax errors or undefined names | |
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics | |
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide | |
flake8 . --count --exit-zero --max-complexity=12 --max-line-length=127 --statistics | |
- name: Install operandi modules | |
run: make install | |
run-operandi-tests: | |
name: Run operandi tests | |
needs: build-native | |
env: | |
ENV_FILE: ./.github/workflows/tests/.env | |
strategy: | |
fail-fast: false | |
matrix: | |
python-version: [ "3.8" ] | |
os: [ ubuntu-latest ] | |
runs-on: ${{ matrix.os }} | |
steps: | |
- name: Checkout repository | |
uses: actions/checkout@v3 | |
with: | |
ref: ${{ github.head_ref }} | |
- name: Set up Python ${{ matrix.python-version }} | |
uses: actions/setup-python@v4 | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: Install operandi dependencies | |
run: | | |
sudo apt-get update && sudo apt-get -y install make | |
python3 -m pip install --upgrade pip setuptools | |
pip3 install -U pip wheel | |
pip install -r tests/requirements.txt | |
- name: Install operandi modules | |
run: make install | |
- name: Import variables from env file | |
shell: bash | |
run: | | |
while read line; do | |
echo "$line" >> $GITHUB_ENV | |
done < ${{ env.ENV_FILE }} | |
- name: copy hpc ssh key | |
env: | |
SSH_KEY_HPC: ${{ secrets.OPERANDI_SSH_KEY_HPC }} | |
run: | | |
mkdir -p /home/runner/.ssh/ | |
echo "$SSH_KEY_HPC" > /home/runner/.ssh/key_hpc | |
chmod 600 /home/runner/.ssh/key_hpc | |
- name: start RabbitMQ Server | |
run: docker compose -f ./docker-compose.yml --env-file ${{ env.ENV_FILE }} up -d operandi-rabbitmq | |
- name: start MongoDB | |
run: docker compose -f ./docker-compose.yml --env-file ${{ env.ENV_FILE }} up -d operandi-mongodb | |
- name: wait starting of RabbitMQ and MongoDB | |
run: sleep 45 | |
- name: run utils tests | |
run: | | |
export $(shell sed 's/=.*//' ${{ env.ENV_FILE }}) | |
pytest -p no:cacheprovider tests/tests_utils/test_*/test_*.py -v | |
- name: run broker tests | |
run: | | |
export $(shell sed 's/=.*//' ${{ env.ENV_FILE }}) | |
pytest -p no:cacheprovider tests/tests_broker/test_*.py -v | |
- name: run server tests | |
run: | | |
export $(shell sed 's/=.*//' ${{ env.ENV_FILE }}) | |
pytest -p no:cacheprovider tests/tests_server/test_*.py -v | |
- name: run harvester tests | |
run: | | |
export $(shell sed 's/=.*//' ${{ env.ENV_FILE }}) | |
pytest -p no:cacheprovider tests/tests_harvester/test_*.py -v | |
- name: run integration tests | |
run: | | |
export $(shell sed 's/=.*//' ${{ env.ENV_FILE }}) | |
pytest -p no:cacheprovider tests/integration_tests/test_*.py -s -v | |
build-and-push-docker-images: | |
name: Push ${{matrix.services.module}} to image registry | |
environment: | |
name: development | |
needs: run-operandi-tests | |
strategy: | |
fail-fast: true | |
matrix: | |
services: [ | |
{ module: "broker", dockerfile: "Dockerfile_broker" }, | |
{ module: "server", dockerfile: "Dockerfile_server" } | |
] | |
os: [ ubuntu-latest ] | |
runs-on: ${{ matrix.os }} | |
permissions: | |
contents: read | |
packages: write | |
steps: | |
- name: Checkout repository | |
uses: actions/checkout@v3 | |
with: | |
ref: ${{ github.head_ref }} | |
- name: Log in to the Container registry | |
uses: docker/login-action@v2 | |
with: | |
registry: ${{ env.REGISTRY }} | |
username: ${{ github.actor }} | |
password: ${{ secrets.GITHUB_TOKEN }} | |
- name: Extract metadata (tags, labels) for Docker | |
uses: docker/metadata-action@v4 | |
id: meta | |
with: | |
github-token: ${{ secrets.GITHUB_TOKEN }} | |
images: ${{ env.REGISTRY }}/${{ env.REPO_NAME }}-${{matrix.services.module}} | |
- name: Build and push Docker image | |
uses: docker/build-push-action@v4 | |
with: | |
context: ./src | |
file: ./src/${{matrix.services.dockerfile}} | |
push: true | |
tags: ${{ steps.meta.outputs.tags }} | |
labels: ${{ steps.meta.outputs.labels }} | |
deploy-development: | |
name: Deploy to development VM | |
environment: | |
name: development | |
needs: build-and-push-docker-images | |
runs-on: ubuntu-latest | |
steps: | |
- name: copy ssh key | |
env: | |
SSH_KEY: ${{ secrets.OPERANDI_SSH_KEY }} | |
run: | | |
mkdir -p ~/.ssh/ | |
echo "$SSH_KEY" > ~/.ssh/key | |
chmod 600 ~/.ssh/key | |
- name: deploy | |
env: | |
SSH_USER: ${{ secrets.OPERANDI_SSH_USER }} | |
SSH_HOST: ${{ secrets.OPERANDI_SSH_HOST_DEV }} | |
run: ssh $SSH_USER@$SSH_HOST -o StrictHostKeyChecking=no -i ~/.ssh/key /home/$SSH_USER/operandi/start-operandi-docker.sh | |
deploy-production: | |
name: Deploy to live VM | |
environment: | |
name: production | |
needs: build-and-push-docker-images | |
runs-on: ubuntu-latest | |
steps: | |
- name: copy ssh key | |
env: | |
SSH_KEY: ${{ secrets.OPERANDI_SSH_KEY }} | |
run: | | |
mkdir -p ~/.ssh/ | |
echo "$SSH_KEY" > ~/.ssh/key | |
chmod 600 ~/.ssh/key | |
- name: deploy | |
env: | |
SSH_USER: ${{ secrets.OPERANDI_SSH_USER }} | |
SSH_HOST: ${{ secrets.OPERANDI_SSH_HOST_PROD }} | |
run: ssh $SSH_USER@$SSH_HOST -o StrictHostKeyChecking=no -i ~/.ssh/key /home/$SSH_USER/start-operandi-docker.sh |