Skip to content
OPERANDI CI/CD

Batch endpoints #509

Sign in to view logs

Batch endpoints

Batch endpoints #509

Workflow file for this run

.github/workflows/ci_cd.yml at b57619d
# For more information see:
# https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
name: OPERANDI CI/CD
on:
push:
# branches: [ main ]
paths:
- .github/**
- src/**
- tests/**
- .*
- docker*
- Makefile
pull_request:
# branches: [ main ]
paths:
- .github/**
- src/**
- tests/**
- .*
- docker*
- Makefile
env:
REGISTRY: ghcr.io
REPO_NAME: ${{ github.repository }}
jobs:
build-native:
name: Native build of Operandi modules
strategy:
fail-fast: true
matrix:
python-version: ["3.8", "3.9", "3.10", "3.11"]
os: [ubuntu-latest]
runs-on: ${{ matrix.os }}
steps:
- name: Checkout repository
uses: actions/checkout@v3
with:
ref: ${{ github.head_ref }}
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install operandi dependencies
run: |
sudo apt-get update && sudo apt-get -y install make
python3 -m pip install --upgrade pip setuptools
pip3 install -U pip wheel
pip install -r tests/requirements.txt
- name: Lint with flake8
run: |
python3 -m pip install flake8
# stop the build if there are Python syntax errors or undefined names
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
flake8 . --count --exit-zero --max-complexity=12 --max-line-length=127 --statistics
- name: Install operandi modules
run: make install
run-operandi-tests:
name: Run operandi tests
needs: build-native
env:
ENV_FILE: ./.github/workflows/tests/.env
strategy:
fail-fast: false
matrix:
python-version: [ "3.8" ]
os: [ ubuntu-latest ]
runs-on: ${{ matrix.os }}
steps:
- name: Checkout repository
uses: actions/checkout@v3
with:
ref: ${{ github.head_ref }}
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install operandi dependencies
run: |
sudo apt-get update && sudo apt-get -y install make
python3 -m pip install --upgrade pip setuptools
pip3 install -U pip wheel
pip install -r tests/requirements.txt
- name: Install operandi modules
run: make install
- name: Import variables from env file
shell: bash
run: |
while read line; do
echo "$line" >> $GITHUB_ENV
done < ${{ env.ENV_FILE }}
- name: copy hpc ssh key
env:
SSH_KEY_HPC: ${{ secrets.OPERANDI_SSH_KEY_HPC }}
run: |
mkdir -p /home/runner/.ssh/
echo "$SSH_KEY_HPC" > /home/runner/.ssh/key_hpc
chmod 600 /home/runner/.ssh/key_hpc
- name: stop RabbitMQ and MongoDB instances if still running
run: docker compose -f ./docker-compose.yml --env-file ${{ env.ENV_FILE }} down --remove-orphans
- name: start RabbitMQ Server
run: docker compose -f ./docker-compose.yml --env-file ${{ env.ENV_FILE }} up -d operandi-rabbitmq
- name: start MongoDB
run: docker compose -f ./docker-compose.yml --env-file ${{ env.ENV_FILE }} up -d operandi-mongodb
- name: run utils tests
run: |
export $(shell sed 's/=.*//' ${{ env.ENV_FILE }})
pytest -p no:cacheprovider tests/tests_utils/test_*/test_*.py -v
- name: run broker tests
run: |
export $(shell sed 's/=.*//' ${{ env.ENV_FILE }})
pytest -p no:cacheprovider tests/tests_broker/test_*.py -v
- name: run server tests
run: |
export $(shell sed 's/=.*//' ${{ env.ENV_FILE }})
pytest -p no:cacheprovider tests/tests_server/test_*.py -v
- name: run harvester tests
run: |
export $(shell sed 's/=.*//' ${{ env.ENV_FILE }})
pytest -p no:cacheprovider tests/tests_harvester/test_*.py -v
- name: run integration tests
run: |
export $(shell sed 's/=.*//' ${{ env.ENV_FILE }})
pytest -p no:cacheprovider tests/integration_tests/test_*.py -s -v
build-and-push-docker-images:
name: Push ${{matrix.services.module}} to image registry
environment:
name: development
needs: run-operandi-tests
strategy:
fail-fast: true
matrix:
services: [
{ module: "broker", dockerfile: "Dockerfile_broker" },
{ module: "server", dockerfile: "Dockerfile_server" }
]
os: [ ubuntu-latest ]
runs-on: ${{ matrix.os }}
permissions:
contents: read
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@v3
with:
ref: ${{ github.head_ref }}
- name: Log in to the Container registry
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata (tags, labels) for Docker
uses: docker/metadata-action@v4
id: meta
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
images: ${{ env.REGISTRY }}/${{ env.REPO_NAME }}-${{matrix.services.module}}
- name: Build and push Docker image
uses: docker/build-push-action@v4
with:
context: ./src
file: ./src/${{matrix.services.dockerfile}}
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
# deploy-development:
# name: Deploy to development VM
# environment:
# name: development
# needs: build-and-push-docker-images
# runs-on: ubuntu-latest
#
# steps:
# - name: copy ssh key
# env:
# SSH_KEY: ${{ secrets.OPERANDI_SSH_KEY }}
# run: |
# mkdir -p ~/.ssh/
# echo "$SSH_KEY" > ~/.ssh/key
# chmod 600 ~/.ssh/key
#
# - name: deploy
# env:
# SSH_USER: ${{ secrets.OPERANDI_SSH_USER }}
# SSH_HOST: ${{ secrets.OPERANDI_SSH_HOST_DEV }}
# run: ssh $SSH_USER@$SSH_HOST -o StrictHostKeyChecking=no -i ~/.ssh/key /home/$SSH_USER/operandi/start-operandi-docker.sh
# deploy-production:
# name: Deploy to live VM
# environment:
# name: production
# needs: build-and-push-docker-images
# runs-on: ubuntu-latest
#
# steps:
# - name: copy ssh key
# env:
# SSH_KEY: ${{ secrets.OPERANDI_SSH_KEY }}
# run: |
# mkdir -p ~/.ssh/
# echo "$SSH_KEY" > ~/.ssh/key
# chmod 600 ~/.ssh/key
#
# - name: deploy
# env:
# SSH_USER: ${{ secrets.OPERANDI_SSH_USER }}
# SSH_HOST: ${{ secrets.OPERANDI_SSH_HOST_PROD }}
# run: ssh $SSH_USER@$SSH_HOST -o StrictHostKeyChecking=no -i ~/.ssh/key /home/$SSH_USER/start-operandi-docker.sh