ci.yaml

name: recap

on:
  push:
    branches: [ main ]

  pull_request:
    branches: [ main ]

jobs:
  unit-tests:
    runs-on: ubuntu-latest

    strategy:
      matrix:
        python-version: ['3.10']

    steps:
    - uses: actions/checkout@v2

    - name: Build using Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    - name: Install dependencies
      run: |
        python3.10 -m venv venv
        source venv/bin/activate
        pip install pdm
        pdm install

    - name: Run black
      run: |
        source venv/bin/activate
        pdm run black --check --diff recap/ tests/

    - name: Run isort
      run: |
        source venv/bin/activate
        pdm run isort recap/ tests/ --check-only --diff

    - name: Run pylint
      run: |
        source venv/bin/activate
        pdm run pylint --fail-under=7.0 recap/ tests/

    - name: Run pyright
      run: |
        source venv/bin/activate
        pdm run pyright

    - name: Test with pytest
      run: |
        source venv/bin/activate
        pdm run pytest tests/unit -vv

  integration-tests:
    runs-on: ubuntu-latest
    needs: unit-tests

    services:
      postgres:
        image: postgres:14
        env:
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: password
          POSTGRES_DB: testdb
        ports:
          - 5432:5432
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

      # TODO Waiting on https://github.com/goccy/bigquery-emulator/pull/208
      bigquery:
        image: ghcr.io/criccomini/bigquery-emulator:0.4.3-envvar
        env:
          BIGQUERY_EMULATOR_PROJECT: test_project
          BIGQUERY_EMULATOR_DATASET: test_dataset
        ports:
          - 9050:9050

      zookeeper:
        image: confluentinc/cp-zookeeper:latest
        ports:
          - 2181:2181
        env:
          ZOOKEEPER_CLIENT_PORT: 2181
        options: >-
          --health-cmd "echo ruok | nc localhost 2181"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

      kafka:
        image: confluentinc/cp-kafka:latest
        ports:
          - 9092:9092
        env:
          KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
          KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
        options: >-
          --health-cmd "kafka-topics --bootstrap-server localhost:9092 --list"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

      schema-registry:
        image: confluentinc/cp-schema-registry:latest
        ports:
          - 8081:8081
        env:
          SCHEMA_REGISTRY_HOST_NAME: schema-registry
          SCHEMA_REGISTRY_LISTENERS: http://schema-registry:8081
          SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper:2181
          SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka:9092
        options: >-
          --health-cmd "curl -f http://schema-registry:8081/"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

      hive-metastore:
        image: ghcr.io/criccomini/hive-metastore-standalone:latest
        ports:
          - 9083:9083

    strategy:
      matrix:
        python-version: ['3.10']

    steps:
    - uses: actions/checkout@v2

    - name: Build using Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    - name: Install dependencies
      run: |
        python3.10 -m venv venv
        source venv/bin/activate
        pip install pdm
        pdm install

    - name: Test with pytest
      run: |
        source venv/bin/activate
        pdm run pytest tests/integration -vv