diff --git a/.github/actions/deploy/action.yaml b/.github/actions/deploy/action.yaml new file mode 100644 index 0000000000..649a323cfb --- /dev/null +++ b/.github/actions/deploy/action.yaml @@ -0,0 +1,66 @@ +name: Common Deployment Steps +description: Implements common deployment steps of all connector workflows +inputs: + connector: + description: Image name of this connector. + required: true + tag_version: + description: Version tag of the connector, such as `v1`. + required: true + tag_sha: + description: Short tag of the commit SHA1, such as 'f32dc1a'. + required: true + variants: + description: Space-separated variant image names of this connector. + default: '' + required: false + +runs: + using: "composite" + steps: + + - name: Push ${{ inputs.connector }} ${{ inputs.variants }} image(s) with commit SHA tag + shell: bash + run: | + for VARIANT in ${{ inputs.connector }} ${{ inputs.variants }}; do + echo "Building and pushing ${VARIANT}:${{ inputs.tag }}..."; + docker build --build-arg BASE_CONNECTOR=ghcr.io/estuary/${{ inputs.connector }}:local \ + --build-arg DOCS_URL=https://go.estuary.dev/${VARIANT} \ + --tag ghcr.io/estuary/${VARIANT}:${{ inputs.tag_sha }} \ + --file connector-variant.Dockerfile .; + docker image push ghcr.io/estuary/${VARIANT}:${{ inputs.tag_sha }}; + done + + - name: Push ${{ inputs.connector }} image(s) with 'dev' and '${{ inputs.tag_version }}' tags + if: ${{ github.event_name == 'push' }} + shell: bash + run: | + for VARIANT in ${{ inputs.connector }} ${{ inputs.variants }}; do + docker image tag ghcr.io/estuary/${VARIANT}:${{ inputs.tag_sha }} ghcr.io/estuary/${VARIANT}:dev; + docker image tag ghcr.io/estuary/${VARIANT}:${{ inputs.tag_sha }} ghcr.io/estuary/${VARIANT}:${{ inputs.tag_version }}; + + docker image push ghcr.io/estuary/${VARIANT}:dev; + docker image push ghcr.io/estuary/${VARIANT}:${{ inputs.tag_version }}; + done + + - name: Install psql + if: ${{ github.event_name == 'push' }} + shell: bash + run: sudo apt install postgresql + + - name: Refresh connector tags for ${{ inputs.connector }} + if: ${{ github.event_name == 'push' }} + shell: bash + env: + PGHOST: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_HOST }} + PGUSER: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_USER }} + PGPASSWORD: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_PASSWORD }} + PGDATABASE: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_DATABASE }} + run: | + for VARIANT in ${{ inputs.connector }} ${{ inputs.variants }}; do + echo "UPDATE connector_tags SET job_status='{\"type\": \"queued\"}' + WHERE connector_id IN ( + SELECT id FROM connectors WHERE image_name='ghcr.io/estuary/${VARIANT}' + ) AND image_tag IN (':${{ inputs.tag_version }}', ':dev');" | psql; + done + diff --git a/.github/actions/setup/action.yaml b/.github/actions/setup/action.yaml new file mode 100644 index 0000000000..d294cc571c --- /dev/null +++ b/.github/actions/setup/action.yaml @@ -0,0 +1,53 @@ +name: Common Setup Steps +description: Implements common setup steps of all connector workflows + +inputs: + setup_gcloud: + description: Whether to set up a gcloud service account. + required: false + default: 'false' + +outputs: + tag_sha: + description: Short tag of the commit SHA1 + value: ${{ steps.determine-sha-tag.outputs.tag }} + +runs: + using: "composite" + steps: + + - name: Determine current SHA tag. + shell: bash + id: determine-sha-tag + run: | + TAG=$(echo $GITHUB_SHA | head -c7) + echo ::set-output name=tag::${TAG} + + - name: Download latest Flow release binaries and add them to $PATH + shell: bash + run: | + ./fetch-flow.sh + echo "${PWD}/flow-bin" >> $GITHUB_PATH + + - name: Login to GitHub package docker registry + shell: bash + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | \ + docker login --username ${{ github.actor }} --password-stdin ghcr.io + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + with: + driver-opts: network=host + + - name: Create docker network flow-test + shell: bash + run: docker network create flow-test + + - name: Set up Cloud SDK + if: ${{ inputs.setup_gcloud_sa }} + uses: google-github-actions/setup-gcloud@v0 + with: + project_id: ${{ secrets.GCP_PROJECT_ID }} + service_account_key: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} + export_default_credentials: true diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2c608ba1ab..be86f5a7a9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -7,9 +7,9 @@ concurrency: on: push: # TODO: Remove jshearer/python_connector_testing before merging - branches: [main, jshearer/python_connector_testing] + branches: [nothing] pull_request: - branches: [main] + branches: [nothing] jobs: build_base_image: @@ -121,9 +121,6 @@ jobs: - connector: source-shopify connector_type: capture python: true - - connector: source-asana - connector_type: capture - python: true - connector: source-airtable connector_type: capture python: true diff --git a/.github/workflows/python-new.yaml b/.github/workflows/python-new.yaml new file mode 100644 index 0000000000..842e2e6fc2 --- /dev/null +++ b/.github/workflows/python-new.yaml @@ -0,0 +1,81 @@ +name: Python-Connectors + +on: + push: + branches: [main] + paths: + - "estuary-cdk/**" + - "source-asana/**" + - "source-google-sheets-native/**" + - "source-hubspot-native/**" + pull_request: + branches: [main] + paths: + - "estuary-cdk/**" + - "source-asana/**" + - "source-google-sheets-native/**" + - "source-hubspot-native/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + py_connector: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + connector: + - name: source-asana + type: capture + version: v1 + - name: source-hubspot-native + type: capture + version: v1 + - name: source-google-sheets-native + type: capture + version: v1 + + steps: + - uses: actions/checkout@v4 + + - id: setup + uses: ./.github/actions/setup + with: + setup_gcloud: false + + - name: Setup Python for ${{ matrix.connector }} + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Poetry + uses: snok/install-poetry@v1 + + - name: Python connector ${{ matrix.connector }} tests + run: | + cd ${{ matrix.connector }} + poetry install + source $(poetry env info --path)/bin/activate + cd .. + pytest . + + - name: Build ${{ matrix.connector }} Python Docker Image + uses: docker/build-push-action@v2 + with: + context: . + file: estuary-cdk/common.Dockerfile + load: true + build-args: | + CONNECTOR_NAME=${{ matrix.connector.name }} + CONNECTOR_TYPE=${{ matrix.connector.type }} + tags: ghcr.io/estuary/${{ matrix.connector }}:local + + - id: deploy + uses: ./.github/actions/deploy + with: + connector: ${{ matrix.connector.name }} + tag_sha: ${{ steps.setup.outputs.tag_sha }} + tag_version: ${{ matrix.connector.version }} + variants: ${{ matrix.connector.variants }} \ No newline at end of file diff --git a/estuary-cdk/common.Dockerfile b/estuary-cdk/common.Dockerfile new file mode 100644 index 0000000000..c51e5b28de --- /dev/null +++ b/estuary-cdk/common.Dockerfile @@ -0,0 +1,37 @@ +# syntax=docker/dockerfile:1 +FROM python:3.12-slim as base +FROM base as builder + +ARG CONNECTOR_NAME + +RUN apt-get update && \ + apt install -y --no-install-recommends \ + python3-poetry + +RUN python -m venv /opt/venv +ENV VIRTUAL_ENV=/opt/venv + +WORKDIR /opt/${CONNECTOR_NAME} +COPY ${CONNECTOR_NAME} /opt/${CONNECTOR_NAME} +COPY estuary-cdk /opt/estuary-cdk + +RUN poetry install + + +FROM base as runner + +ARG CONNECTOR_NAME +ARG CONNECTOR_TYPE +ARG DOCS_URL + +LABEL FLOW_RUNTIME_PROTOCOL=${CONNECTOR_TYPE} +LABEL FLOW_RUNTIME_CODEC=json + +COPY --from=builder /opt/$CONNECTOR_NAME /opt/$CONNECTOR_NAME +COPY --from=builder /opt/estuary-cdk /opt/estuary-cdk +COPY --from=builder /opt/venv /opt/venv + +ENV DOCS_URL=${DOCS_URL} +ENV CONNECTOR_NAME=${CONNECTOR_NAME} + +CMD /opt/venv/bin/python -m $(echo "$CONNECTOR_NAME" | tr '-' '_') \ No newline at end of file