SNOW-1658905 Integration tests setup for Iceberg #1609
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Kafka Connector Integration Test | |
| on: | |
| push: | |
| branches: [ master ] | |
| pull_request: | |
| branches: '**' | |
| jobs: | |
| build_and_test: | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstrategyfail-fast | |
| matrix: | |
| snowflake_cloud: [ 'AWS', 'AZURE', 'GCS' ] | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v2 | |
| - name: "Install Java 8" | |
| uses: actions/setup-java@v1 | |
| with: | |
| java-version: 1.8 | |
| - name: "Cache local Maven repository" | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-maven- | |
| - name: Install Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.9' | |
| architecture: 'x64' | |
| - name: Decrypt profile.json in Snowflake Cloud ${{ matrix.snowflake_cloud }} | |
| run: ./.github/scripts/decrypt_secret.sh ${{ matrix.snowflake_cloud }} | |
| env: | |
| SNOWFLAKE_TEST_PROFILE_SECRET: ${{ secrets.SNOWFLAKE_TEST_PROFILE_SECRET }} | |
| - name: Install Dependency | |
| run: | | |
| pip3 install --upgrade setuptools | |
| pip3 install requests certifi "confluent-kafka[avro,json,protobuf]==1.9.2" | |
| pip3 install avro kafka-python | |
| pip3 install protobuf | |
| pip3 install --upgrade snowflake-connector-python==2.7.4 | |
| curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash | |
| sudo apt-get -y install jq vim | |
| sudo apt-get -y install protobuf-compiler | |
| - name: Install Squid as Proxy Server and Apache Utils for Password Authentication | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get -y install squid3 | |
| sudo apt-get install apache2-utils | |
| - name: Change squid config and run Proxy Server | |
| run: | | |
| sudo touch /etc/squid/passwords | |
| sudo chmod 777 /etc/squid/passwords | |
| sudo htpasswd -db -c /etc/squid/passwords admin test | |
| sudo mv .github/scripts/squid.conf /etc/squid/squid.conf | |
| sudo service squid start | |
| - name: Build with Unit and Integration Test against Snowflake Cloud in ${{ matrix.snowflake_cloud }} | |
| env: | |
| JACOCO_COVERAGE: true | |
| SNOWFLAKE_CREDENTIAL_FILE: "../profile.json" | |
| SHELL: "/bin/bash" | |
| run: | | |
| cd test | |
| ./build_runtime_jar.sh ../../snowflake-kafka-connector | |
| # Commented out because the build was very unstable | |
| # Error that might be a hint: {'detail': ErrorDetail(string='Rate limit reached. Please upload with the Codecov repository upload token to resolve issue. Expected available in 383 seconds.', code='throttled')} | |
| # Consider migrating to newer version (v4) | |
| # - name: Code Coverage | |
| # uses: codecov/codecov-action@v1 |