From 454b31d29849447dc939a8ec7ed3414f54e47050 Mon Sep 17 00:00:00 2001 From: Johnny Graettinger Date: Thu, 29 Feb 2024 23:24:20 +0000 Subject: [PATCH] TEMP remove ci.yaml while working on python CI --- .github/workflows/ci.yaml | 332 -------------------------------------- 1 file changed, 332 deletions(-) delete mode 100644 .github/workflows/ci.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml deleted file mode 100644 index 23194d1267..0000000000 --- a/.github/workflows/ci.yaml +++ /dev/null @@ -1,332 +0,0 @@ -name: CI - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - push: - # TODO: Remove jshearer/python_connector_testing before merging - branches: [main, jshearer/python_connector_testing] - pull_request: - branches: [main] - -jobs: - build_base_image: - runs-on: ubuntu-20.04 - strategy: - fail-fast: false - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Prepare - id: prep - run: | - TAG=$(echo $GITHUB_SHA | head -c7) - echo ::set-output name=tag::${TAG} - VERSION=$(cat base-image/VERSION | tr -d '\n') - echo ::set-output name=version::${VERSION} - - - name: Login to GitHub package docker registry - run: | - echo "${{ secrets.GITHUB_TOKEN }}" | \ - docker login --username ${{ github.actor }} --password-stdin ghcr.io - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - with: - driver-opts: | - network=host - - - name: Build base-image Docker Image - uses: docker/build-push-action@v2 - with: - context: . - file: base-image/Dockerfile - load: true - tags: ghcr.io/estuary/base-image:local - - - name: Push base-image image - uses: docker/build-push-action@v2 - with: - context: . - file: base-image/Dockerfile - push: true - tags: ghcr.io/estuary/base-image:${{ steps.prep.outputs.tag }} - - - name: Push base-image image with 'dev' tag - if: ${{ github.event_name == 'push' }} - uses: docker/build-push-action@v2 - with: - context: . - file: base-image/Dockerfile - push: true # See 'if' above - tags: ghcr.io/estuary/base-image:dev,ghcr.io/estuary/base-image:${{ steps.prep.outputs.version }} - - build_connectors: - runs-on: ubuntu-20.04 - needs: build_base_image - strategy: - fail-fast: false - matrix: - connector: - - source-alpaca - - source-bigquery-batch - - source-dynamodb - - source-firestore - - source-gcs - - source-google-drive - - source-hello-world - - source-http-file - - source-http-ingest - - source-kafka - - source-kinesis - - source-mongodb - - source-mysql - - source-mysql-batch - - source-postgres - - source-postgres-batch - - source-redshift-batch - - source-s3 - - source-sftp - - source-snowflake - - source-sqlserver - - source-test - - materialize-bigquery - - materialize-databricks - - materialize-dynamodb - - materialize-elasticsearch - - materialize-firebolt - - materialize-google-pubsub - - materialize-google-sheets - - materialize-mongodb - - materialize-motherduck - - materialize-mysql - - materialize-pinecone - - materialize-postgres - - materialize-redshift - - materialize-rockset - - materialize-s3-parquet - - materialize-snowflake - - materialize-starburst - - materialize-sqlite - - materialize-webhook - - materialize-sqlserver - include: - - connector: source-criteo - connector_type: capture - python: true - - connector: source-shopify - connector_type: capture - python: true - - connector: source-airtable - connector_type: capture - python: true - - connector: source-google-ads - connector_type: capture - python: true - - connector: source-facebook-marketing - connector_type: capture - python: true - - connector: source-hubspot - connector_type: capture - python: true - - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Prepare - id: prep - run: | - TAG=$(echo $GITHUB_SHA | head -c7) - echo ::set-output name=tag::${TAG} - VERSION=$(cat ${{ matrix.connector }}/VERSION | tr -d '\n') - echo ::set-output name=version::${VERSION} - VARIANTS="${{ matrix.connector }} $(cat ${{ matrix.connector }}/VARIANTS 2>/dev/null || true)" - echo ::set-output name=variants::${VARIANTS} - - - name: Download latest Flow release binaries and add them to $PATH - run: | - ./fetch-flow.sh - echo "${PWD}/flow-bin" >> $GITHUB_PATH - - - name: Set up Cloud SDK - if: matrix.connector == 'source-gcs' || matrix.python - uses: google-github-actions/setup-gcloud@v0 - with: - project_id: ${{ secrets.GCP_PROJECT_ID }} - service_account_key: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} - export_default_credentials: true - - - name: Login to GitHub package docker registry - run: | - echo "${{ secrets.GITHUB_TOKEN }}" | \ - docker login --username ${{ github.actor }} --password-stdin ghcr.io - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - with: - driver-opts: | - network=host - - - name: Create docker network flow-test - run: docker network create flow-test - - - name: Start Dockerized test infrastructure - if: | - contains(fromJson('[ - "materialize-dynamodb", - "materialize-elasticsearch", - "source-dynamodb", - "source-kinesis", - "source-mysql", - "source-postgres", - "source-sftp", - "source-sqlserver", - "source-mongodb" - ]'), matrix.connector) - run: | - docker compose --file ${{ matrix.connector }}/docker-compose.yaml up --wait - docker logs ${{ matrix.connector }}-db-1 - - - name: Test setup - if: matrix.connector == 'source-mongodb' - run: | - bash source-mongodb/docker-compose-init.sh - - - name: Build ${{ matrix.connector }} Docker Image - uses: docker/build-push-action@v2 - if: ${{!matrix.python}} - with: - context: . - file: ${{ matrix.connector }}/Dockerfile - load: true - build-args: BASE_IMAGE=ghcr.io/estuary/base-image:${{ steps.prep.outputs.tag }} - tags: ghcr.io/estuary/${{ matrix.connector }}:local - secrets: | - "rockset_api_key=${{ secrets.ROCKSET_API_KEY }}" - - - name: Build ${{ matrix.connector }} Python Docker Image - uses: docker/build-push-action@v2 - if: ${{matrix.python}} - with: - context: . - file: python/Dockerfile - load: true - build-args: | - CONNECTOR_NAME=${{matrix.connector}} - CONNECTOR_TYPE=${{matrix.connector_type}} - tags: ghcr.io/estuary/${{ matrix.connector }}:local - - - name: Setup Python for ${{ matrix.connector }} - uses: actions/setup-python@v5 - if: ${{matrix.python}} - with: - python-version: '3.11' - - - name: Install Poetry - if: ${{matrix.python}} - uses: snok/install-poetry@v1 - - - name: Python connector ${{ matrix.connector }} snapshot tests - if: ${{matrix.python}} - run: | - cd ${{matrix.connector}}; - poetry install; - source $(poetry env info --path)/bin/activate - cd ..; - pytest ${{matrix.connector}}/tests; - - - name: Start Dockerized test infrastructure - if: matrix.connector == 'source-kafka' - run: | - docker compose --file infra/docker-compose.yaml up --wait - - - name: Source connector ${{ matrix.connector }} integration tests - if: | - contains(fromJson('[ - "source-dynamodb", - "source-gcs", - "source-kinesis", - "source-mysql", - "source-postgres", - "source-kafka", - "source-s3", - "source-sftp", - "source-sqlserver" - ]'), matrix.connector) - env: - GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} - ROCKSET_API_KEY: ${{ secrets.ROCKSET_API_KEY }} - MYSQL_DATABASE: test - - run: CONNECTOR=${{ matrix.connector }} VERSION=local ./tests/run.sh; - - - name: Materialization connector ${{ matrix.connector }} integration tests - if: | - contains(fromJson('[ - "materialize-dynamodb", - "materialize-elasticsearch", - "materialize-google-sheets", - "materialize-pinecone", - "materialize-postgres", - "materialize-mysql", - "materialize-s3-parquet", - "materialize-sqlserver", - "materialize-mongodb" - ]'), matrix.connector) - env: - GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} - PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - PINECONE_INDEX: ${{ vars.PINECONE_INDEX }} - PINECONE_PROJECT_ID: ${{ vars.PINECONE_PROJECT_ID }} - PINECONE_ENVIRONMENT: ${{ vars.PINECONE_ENVIRONMENT }} - - run: CONNECTOR=${{ matrix.connector }} VERSION=local tests/materialize/run.sh; - - - name: Push ${{ matrix.connector }} image(s) with commit SHA tag - run: | - for VARIANT in ${{ steps.prep.outputs.variants }}; do - echo "Building and pushing ${VARIANT}:${{ steps.prep.outputs.tag }}..."; - docker build --build-arg BASE_CONNECTOR=ghcr.io/estuary/${{ matrix.connector }}:local \ - --build-arg DOCS_URL=https://go.estuary.dev/${VARIANT} \ - --tag ghcr.io/estuary/${VARIANT}:${{ steps.prep.outputs.tag }} \ - --file connector-variant.Dockerfile .; - docker image push ghcr.io/estuary/${VARIANT}:${{ steps.prep.outputs.tag }}; - done - - - name: Push ${{ matrix.connector }} image(s) with 'dev' and '${{ steps.prep.outputs.version }}' tags - if: ${{ github.event_name == 'push' }} - run: | - for VARIANT in ${{ steps.prep.outputs.variants }}; do - docker image tag ghcr.io/estuary/${VARIANT}:${{ steps.prep.outputs.tag }} ghcr.io/estuary/${VARIANT}:dev; - docker image tag ghcr.io/estuary/${VARIANT}:${{ steps.prep.outputs.tag }} ghcr.io/estuary/${VARIANT}:${{ steps.prep.outputs.version }}; - docker image push ghcr.io/estuary/${VARIANT}:dev; - docker image push ghcr.io/estuary/${VARIANT}:${{ steps.prep.outputs.version }}; - done - - - name: Install psql - if: ${{ github.event_name == 'push' }} - run: | - sudo apt update - sudo apt install postgresql - - - name: Refresh connector tags for ${{ matrix.connector }} - if: ${{ github.event_name == 'push' }} - env: - PGHOST: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_HOST }} - PGUSER: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_USER }} - PGPASSWORD: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_PASSWORD }} - PGDATABASE: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_DATABASE }} - run: | - for VARIANT in ${{ steps.prep.outputs.variants }}; do - echo "UPDATE connector_tags SET job_status='{\"type\": \"queued\"}' - WHERE connector_id IN ( - SELECT id FROM connectors WHERE image_name='ghcr.io/estuary/${VARIANT}' - ) AND image_tag IN (':${{ steps.prep.outputs.version }}', ':dev');" | psql; - done