ci_cd.yml

# For more information see:
# https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: OPERANDI CI/CD

on:
  push:
    # branches: [ main ]
    paths:
      - .github/**
      - src/**
      - tests/**
      - .*
      - docker*
      - Makefile
  pull_request:
    # branches: [ main ]
    paths:
      - .github/**
      - src/**
      - tests/**
      - .*
      - docker*
      - Makefile

env:
  REGISTRY: ghcr.io
  REPO_NAME: ${{ github.repository }}

jobs:
  build-native:
    name: Native build of Operandi modules
    strategy:
      fail-fast: true
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11"]
        os: [ubuntu-latest]
    runs-on: ${{ matrix.os }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
        with:
          ref: ${{ github.head_ref }}

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install operandi dependencies
        run: |
          sudo apt-get update && sudo apt-get -y install make
          python3 -m pip install --upgrade pip setuptools  
          pip3 install -U pip wheel
          pip install -r tests/requirements.txt

      - name: Lint with flake8
        run: |
          python3 -m pip install flake8      
          # stop the build if there are Python syntax errors or undefined names
          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
          flake8 . --count --exit-zero --max-complexity=12 --max-line-length=127 --statistics

      - name: Install operandi modules
        run: make install

  run-operandi-tests:
    name: Run operandi tests
    needs: build-native
    env:
      ENV_FILE: ./.github/workflows/tests/.env
    strategy:
      fail-fast: false
      matrix:
        python-version: [ "3.8" ]
        os: [ ubuntu-latest ]
    runs-on: ${{ matrix.os }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
        with:
          ref: ${{ github.head_ref }}

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install operandi dependencies
        run: |
          sudo apt-get update && sudo apt-get -y install make
          python3 -m pip install --upgrade pip setuptools  
          pip3 install -U pip wheel
          pip install -r tests/requirements.txt

      - name: Install operandi modules
        run: make install

      - name: Import variables from env file
        shell: bash
        run: |
          while read line; do
            echo "$line" >> $GITHUB_ENV
          done < ${{ env.ENV_FILE }}

      - name: copy hpc ssh key
        env:
          SSH_KEY_HPC: ${{ secrets.OPERANDI_SSH_KEY_HPC }}
        run: |
          mkdir -p /home/runner/.ssh/
          echo "$SSH_KEY_HPC" > /home/runner/.ssh/key_hpc
          chmod 600 /home/runner/.ssh/key_hpc

      - name: start RabbitMQ Server
        run: docker compose -f ./docker-compose.yml --env-file ${{ env.ENV_FILE }} up -d operandi-rabbitmq

      - name: start MongoDB
        run: docker compose -f ./docker-compose.yml --env-file ${{ env.ENV_FILE }} up -d operandi-mongodb

      - name: wait starting of RabbitMQ and MongoDB
        run: sleep 45

      - name: run utils tests
        run: |
          export $(shell sed 's/=.*//' ${{ env.ENV_FILE }})
          pytest -p no:cacheprovider tests/tests_utils/test_*/test_*.py -v

      - name: run broker tests
        run: |
          export $(shell sed 's/=.*//' ${{ env.ENV_FILE }})
          pytest -p no:cacheprovider tests/tests_broker/test_*.py -v

      - name: run server tests
        run: |
          export $(shell sed 's/=.*//' ${{ env.ENV_FILE }})
          pytest -p no:cacheprovider tests/tests_server/test_*.py -v

      - name: run harvester tests
        run: |
          export $(shell sed 's/=.*//' ${{ env.ENV_FILE }})
          pytest -p no:cacheprovider tests/tests_harvester/test_*.py -v

      - name: run integration tests
        run: |
          export $(shell sed 's/=.*//' ${{ env.ENV_FILE }})
          pytest -p no:cacheprovider tests/integration_tests/test_*.py -s -v

  build-and-push-docker-images:
    name: Push ${{matrix.services.module}} to image registry
    environment:
      name: development
    needs: run-operandi-tests
    strategy:
      fail-fast: true
      matrix:
        services: [
          { module: "broker", dockerfile: "Dockerfile_broker" },
          { module: "server", dockerfile: "Dockerfile_server" }
        ]
        os: [ ubuntu-latest ]
    runs-on: ${{ matrix.os }}

    permissions:
      contents: read
      packages: write

    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
        with:
          ref: ${{ github.head_ref }}

      - name: Log in to the Container registry
        uses: docker/login-action@v2
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Extract metadata (tags, labels) for Docker
        uses: docker/metadata-action@v4
        id: meta
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          images: ${{ env.REGISTRY }}/${{ env.REPO_NAME }}-${{matrix.services.module}}

      - name: Build and push Docker image
        uses: docker/build-push-action@v4
        with:
          context: ./src
          file: ./src/${{matrix.services.dockerfile}}
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }} 

  deploy-development:
    name: Deploy to development VM
    environment:
      name: development
    needs: build-and-push-docker-images
    runs-on: ubuntu-latest

    steps:
      - name: copy ssh key
        env:
          SSH_KEY: ${{ secrets.OPERANDI_SSH_KEY }}
        run: |
          mkdir -p ~/.ssh/
          echo "$SSH_KEY" > ~/.ssh/key
          chmod 600 ~/.ssh/key

      - name: deploy
        env:
          SSH_USER: ${{ secrets.OPERANDI_SSH_USER }}
          SSH_HOST: ${{ secrets.OPERANDI_SSH_HOST_DEV }}
        run: ssh $SSH_USER@$SSH_HOST -o StrictHostKeyChecking=no -i ~/.ssh/key /home/$SSH_USER/operandi/start-operandi-docker.sh

  deploy-production:
    name: Deploy to live VM
    environment:
      name: production
    needs: build-and-push-docker-images
    runs-on: ubuntu-latest

    steps:
      - name: copy ssh key
        env:
          SSH_KEY: ${{ secrets.OPERANDI_SSH_KEY }}
        run: |
          mkdir -p ~/.ssh/
          echo "$SSH_KEY" > ~/.ssh/key
          chmod 600 ~/.ssh/key

      - name: deploy
        env:
          SSH_USER: ${{ secrets.OPERANDI_SSH_USER }}
          SSH_HOST: ${{ secrets.OPERANDI_SSH_HOST_PROD }}
        run: ssh $SSH_USER@$SSH_HOST -o StrictHostKeyChecking=no -i ~/.ssh/key /home/$SSH_USER/start-operandi-docker.sh