py-tests.yml

#  Copyright 2021 Collate
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#  http://www.apache.org/licenses/LICENSE-2.0
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

name: py-tests
on:
  push:
    branches:
      - main
      - '0.[0-9]+.[0-9]+'
    paths-ignore:
      - 'openmetadata-docs/**'
      - 'openmetadata-docs-v1/**'
  pull_request_target:
    types: [labeled, opened, synchronize, reopened]
    paths:
      - "ingestion/**"
      - "openmetadata-service/**"
      - "openmetadata-spec/src/main/resources/json/schema/**"
      - "pom.xml"
      - "Makefile"

permissions:
  contents: read

jobs:
  py-run-tests:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        py-version: ['3.7', '3.8', '3.9']
    steps:

    - name: Wait for the labeler
      uses: lewagon/wait-on-check-action@0179dfc359f90a703c41240506f998ee1603f9ea  #v1.0.0
      if: ${{ github.event_name == 'pull_request_target' }}
      with:
        ref: ${{ github.event.pull_request.head.sha }}
        check-name: Team Label
        repo-token: ${{ secrets.GITHUB_TOKEN }}
        wait-interval: 90

    - name: Verify PR labels
      uses: jesusvasquez333/verify-pr-label-action@v1.4.0
      if: ${{ github.event_name == 'pull_request_target' }}
      with:
        github-token: '${{ secrets.GITHUB_TOKEN }}'
        valid-labels: 'safe to test'
        pull-request-number: '${{ github.event.pull_request.number }}'
        disable-reviews: true  # To not auto approve changes

    - name: Checkout
      uses: actions/checkout@v3
      with:
        ref: ${{ github.event.pull_request.head.sha }}
        fetch-depth: 0

    - name: Set up JDK 11
      uses: actions/setup-java@v3
      with:
        java-version: '11'
        distribution: 'adopt'

    - name: Set up Python ${{ matrix.py-version }}
      uses: actions/setup-python@v4
      with:
        python-version:  ${{ matrix.py-version }}

    - name: Install Ubuntu dependencies
      run: |
        sudo apt-get update && sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
        unixodbc-dev libevent-dev python3-dev libkrb5-dev

    - name: Generate models
      run: |
        python3 -m venv env
        source env/bin/activate
        sudo make install_antlr_cli
        make install_dev generate

    - name: Install open-metadata dependencies
      run: |
        source env/bin/activate
        make install_all install_test

    - name: Start Server and Ingest Sample Data
      uses: nick-fields/retry@v2.8.3
      env:
        INGESTION_DEPENDENCY: "mysql,elasticsearch"
      with:
        timeout_minutes: 30
        max_attempts: 2
        retry_on: error
        command: ./docker/run_local_docker.sh -m no-ui

    - name: Run Python Tests
      if: ${{ matrix.py-version != '3.9' }}
      run: |
        source env/bin/activate
        make run_python_tests

    - name: Run Python Tests & record coverage
      if: ${{ matrix.py-version == '3.9' }}
      run: |
        source env/bin/activate
        make coverage
        rm pom.xml
        # fix coverage xml report for github
        sed -i 's/src\/metadata/\/github\/workspace\/ingestion\/src\/metadata/g' ingestion/ci-coverage.xml

    - name: Clean Up
      run: |
        cd ./docker/development
        docker compose down --remove-orphans
        sudo rm -rf ${PWD}/docker-volume

      # we have to pass these args values since we are working with the 'pull_request_target' trigger
    - name: Push Results in PR to Sonar
      id: push-to-sonar
      if: ${{ github.event_name == 'pull_request_target' && matrix.py-version == '3.9' }}
      continue-on-error: true
      uses: sonarsource/sonarcloud-github-action@master
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        SONAR_TOKEN: ${{ secrets.INGESTION_SONAR_SECRET }}
      with:
        projectBaseDir: ingestion/
        args: >
          -Dproject.settings=ingestion/sonar-project.properties
          -Dsonar.pullrequest.key=${{ github.event.pull_request.number }}
          -Dsonar.pullrequest.branch=${{ github.head_ref }}
          -Dsonar.pullrequest.github.repository=OpenMetadata
          -Dsonar.scm.revision=${{ github.event.pull_request.head.sha }}
          -Dsonar.pullrequest.provider=github

    # next two steps are for retrying "Push Results in PR to Sonar" step in case it fails
    - name: Wait to retry 'Push Results in PR to Sonar'
      if: ${{ github.event_name == 'pull_request_target' && matrix.py-version == '3.9' && steps.push-to-sonar.outcome != 'success' }}
      run: sleep 20s
      shell: bash

    - name: Retry 'Push Results in PR to Sonar'
      uses: sonarsource/sonarcloud-github-action@master
      if: ${{ github.event_name == 'pull_request_target' && matrix.py-version == '3.9' && steps.push-to-sonar.outcome != 'success' }}
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        SONAR_TOKEN: ${{ secrets.INGESTION_SONAR_SECRET }}
      with:
        projectBaseDir: ingestion/
        args: >
          -Dproject.settings=ingestion/sonar-project.properties
          -Dsonar.pullrequest.key=${{ github.event.pull_request.number }}
          -Dsonar.pullrequest.branch=${{ github.head_ref }}
          -Dsonar.pullrequest.github.repository=OpenMetadata
          -Dsonar.scm.revision=${{ github.event.pull_request.head.sha }}
          -Dsonar.pullrequest.provider=github