End2EndTestApache.yml

name: Kafka Connector Apache End2End Test

on:
  push:
    branches: [ master ]
  pull_request:
    branches: '**'

jobs:
  build_and_test:
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstrategyfail-fast
      matrix:
        apache_kafka_version: [ '2.5.1', '2.8.1', '3.2.1' ]
        snowflake_cloud: [ 'AWS', 'AZURE', 'GCS' ]
        java_test_version: [ '8', '11', '17' ]
    steps:
    - name: Checkout Code
      uses: actions/checkout@v2
    - name: "Install Java 8"
      uses: actions/setup-java@v3
      with:
        distribution: 'zulu'
        java-version: 8
    - name: "Cache local Maven repository"
      uses: actions/cache@v2
      with:
        path: ~/.m2/repository
        key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
        restore-keys: |
          ${{ runner.os }}-maven-
    - name: Install Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.9'
        architecture: 'x64'
    - name: Decrypt profile.json in Snowflake Cloud ${{ matrix.snowflake_cloud }}
      run: ./.github/scripts/decrypt_secret.sh ${{ matrix.snowflake_cloud }}
      env:
        SNOWFLAKE_TEST_PROFILE_SECRET: ${{ secrets.SNOWFLAKE_TEST_PROFILE_SECRET }}
    - name: Install Dependency
      run: |
        pip3 install --upgrade setuptools
        sudo apt-get install librdkafka-dev
        pip3 install requests certifi "confluent-kafka[avro,json,protobuf]==1.9.2"
        pip3 install avro kafka-python
        pip3 install --upgrade protobuf==3.20.3
        pip3 install --upgrade snowflake-connector-python==2.7.4
        curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
        sudo apt-get -y install jq vim
        sudo apt-get -y install protobuf-compiler

    - name: Install Squid as Proxy Server and Apache Utils for Password Authentication
      run: |
        sudo apt-get update
        sudo apt-get -y install squid3
        sudo apt-get install apache2-utils

    - name: Change squid config and run Proxy Server
      run: |
        sudo touch /etc/squid/passwords
        sudo chmod 777 /etc/squid/passwords
        sudo htpasswd -db -c /etc/squid/passwords admin test
        sudo mv .github/scripts/squid.conf /etc/squid/squid.conf
        sudo service squid start
    
    - name: Build with Unit Test for Apache
      env:
        JACOCO_COVERAGE: true
        SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"
        SHELL: "/bin/bash"
      run: |
        cd test
        ./build_runtime_jar.sh ../../snowflake-kafka-connector package

    - name: Setup JAVA for tests version ${{ matrix.java_test_version }}
      uses: actions/setup-java@v3
      with:
        distribution: 'zulu'
        java-version: ${{ matrix.java_test_version }}

    - name: End to End Test of Apache Platform ${{ matrix.apache_kafka_version }} against Snowflake in ${{ matrix.snowflake_cloud }} with JAVA ${{ matrix.java_test_version }}
      env:
        SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"
        SF_CLOUD_PLATFORM: ${{ matrix.snowflake_cloud }}
        # Will not run delivery guarantee tests in GCS (Check runDeliveryGuaranteeTests in test_verify.py)
        ENABLE_DELIVERY_GUARANTEE_TESTS: "True"
      run: |
        cd test
        ./run_test_apache.sh ${{ matrix.apache_kafka_version }} ./apache_properties