-
Notifications
You must be signed in to change notification settings - Fork 17
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
introduce new github workflow for python connectors
Begin to factor out common setup steps into reuseable composite actions. Add a common estuary-cdk Dockerfile
- Loading branch information
1 parent
951a0ba
commit 1360a60
Showing
5 changed files
with
259 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,78 @@ | ||
name: Common Deployment Steps | ||
description: Implements common deployment steps of all connector workflows | ||
inputs: | ||
connector: | ||
description: Image name of this connector. | ||
required: true | ||
pg_database: | ||
description: Postgres database to use. | ||
required: true | ||
pg_host: | ||
description: Postgres host to use. | ||
required: true | ||
pg_password: | ||
description: Postgres password to use. | ||
required: true | ||
pg_user: | ||
description: Postgres user to use. | ||
required: true | ||
tag_sha: | ||
description: Short tag of the commit SHA1, such as 'f32dc1a'. | ||
required: true | ||
tag_version: | ||
description: Version tag of the connector, such as `v1`. | ||
required: true | ||
variants: | ||
description: Space-separated variant image names of this connector. | ||
default: '' | ||
required: false | ||
|
||
runs: | ||
using: "composite" | ||
steps: | ||
|
||
- name: Push ${{ inputs.connector }} ${{ inputs.variants }} image(s) with commit SHA tag | ||
shell: bash | ||
run: | | ||
for VARIANT in ${{ inputs.connector }} ${{ inputs.variants }}; do | ||
echo "Building and pushing ${VARIANT}:${{ inputs.tag }}..."; | ||
docker build --build-arg BASE_CONNECTOR=ghcr.io/estuary/${{ inputs.connector }}:local \ | ||
--build-arg DOCS_URL=https://go.estuary.dev/${VARIANT} \ | ||
--tag ghcr.io/estuary/${VARIANT}:${{ inputs.tag_sha }} \ | ||
--file connector-variant.Dockerfile .; | ||
docker image push ghcr.io/estuary/${VARIANT}:${{ inputs.tag_sha }}; | ||
done | ||
- name: Push ${{ inputs.connector }} image(s) with 'dev' and '${{ inputs.tag_version }}' tags | ||
if: ${{ github.event_name == 'push' }} | ||
shell: bash | ||
run: | | ||
for VARIANT in ${{ inputs.connector }} ${{ inputs.variants }}; do | ||
docker image tag ghcr.io/estuary/${VARIANT}:${{ inputs.tag_sha }} ghcr.io/estuary/${VARIANT}:dev; | ||
docker image tag ghcr.io/estuary/${VARIANT}:${{ inputs.tag_sha }} ghcr.io/estuary/${VARIANT}:${{ inputs.tag_version }}; | ||
docker image push ghcr.io/estuary/${VARIANT}:dev; | ||
docker image push ghcr.io/estuary/${VARIANT}:${{ inputs.tag_version }}; | ||
done | ||
- name: Install psql | ||
if: ${{ github.event_name == 'push' }} | ||
shell: bash | ||
run: sudo apt install postgresql | ||
|
||
- name: Refresh connector tags for ${{ inputs.connector }} | ||
if: ${{ github.event_name == 'push' }} | ||
shell: bash | ||
env: | ||
PGDATABASE: ${{ inputs.pg_database }} | ||
PGHOST: ${{ inputs.pg_host }} | ||
PGPASSWORD: ${{ inputs.pg_password }} | ||
PGUSER: ${{ inputs.pg_user }} | ||
run: | | ||
for VARIANT in ${{ inputs.connector }} ${{ inputs.variants }}; do | ||
echo "UPDATE connector_tags SET job_status='{\"type\": \"queued\"}' | ||
WHERE connector_id IN ( | ||
SELECT id FROM connectors WHERE image_name='ghcr.io/estuary/${VARIANT}' | ||
) AND image_tag IN (':${{ inputs.tag_version }}', ':dev');" | psql; | ||
done | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,56 @@ | ||
name: Common Setup Steps | ||
description: Implements common setup steps of all connector workflows | ||
|
||
inputs: | ||
github_token: | ||
description: GitHub token to use. | ||
required: true | ||
gcp_project_id: | ||
description: GCP Project for which to set up a service account. | ||
gcp_service_account: | ||
description: GCP Service account. | ||
|
||
outputs: | ||
tag_sha: | ||
description: Short tag of the commit SHA1 | ||
value: ${{ steps.determine-sha-tag.outputs.tag }} | ||
|
||
runs: | ||
using: "composite" | ||
steps: | ||
|
||
- name: Determine current SHA tag. | ||
shell: bash | ||
id: determine-sha-tag | ||
run: | | ||
TAG=$(echo $GITHUB_SHA | head -c7) | ||
echo "tag=${TAG}" >> $GITHUB_OUTPUT | ||
- name: Download latest Flow release binaries and add them to $PATH | ||
shell: bash | ||
run: | | ||
./fetch-flow.sh | ||
echo "${PWD}/flow-bin" >> $GITHUB_PATH | ||
- name: Login to GitHub package docker registry | ||
shell: bash | ||
run: | | ||
echo "${{ inputs.github_token }}" | \ | ||
docker login --username ${{ github.actor }} --password-stdin ghcr.io | ||
- name: Set up Docker Buildx | ||
uses: docker/setup-buildx-action@v1 | ||
with: | ||
driver-opts: network=host | ||
|
||
- name: Create docker network flow-test | ||
shell: bash | ||
run: docker network create flow-test | ||
|
||
- name: Set up Cloud SDK | ||
if: ${{ inputs.gcp_project_id }} | ||
uses: google-github-actions/setup-gcloud@v0 | ||
with: | ||
project_id: ${{ inputs.gcp_project_id }} | ||
service_account_key: ${{ inputs.gcp_service_account }} | ||
export_default_credentials: true |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,86 @@ | ||
name: Python-Connectors | ||
|
||
on: | ||
push: | ||
branches: [main] | ||
paths: | ||
- "estuary-cdk/**" | ||
- "source-asana/**" | ||
- "source-google-sheets-native/**" | ||
- "source-hubspot-native/**" | ||
pull_request: | ||
branches: [main] | ||
paths: | ||
- "estuary-cdk/**" | ||
- "source-asana/**" | ||
- "source-google-sheets-native/**" | ||
- "source-hubspot-native/**" | ||
|
||
concurrency: | ||
group: ${{ github.workflow }}-${{ github.ref }} | ||
cancel-in-progress: true | ||
|
||
jobs: | ||
py_connector: | ||
runs-on: ubuntu-latest | ||
strategy: | ||
fail-fast: false | ||
matrix: | ||
connector: | ||
- name: source-asana | ||
type: capture | ||
version: v1 | ||
- name: source-hubspot-native | ||
type: capture | ||
version: v1 | ||
- name: source-google-sheets-native | ||
type: capture | ||
version: v1 | ||
|
||
steps: | ||
- uses: actions/checkout@v4 | ||
|
||
- name: Common Setup | ||
id: setup | ||
uses: ./.github/actions/setup | ||
with: | ||
github_token: ${{ secrets.GITHUB_TOKEN }} | ||
|
||
- name: Install Python | ||
uses: actions/setup-python@v5 | ||
with: | ||
python-version: "3.12" | ||
|
||
- name: Install Poetry | ||
uses: snok/install-poetry@v1 | ||
|
||
- name: ${{ matrix.connector.name }} tests | ||
run: | | ||
cd ${{ matrix.connector.name }} | ||
poetry install | ||
source $(poetry env info --path)/bin/activate | ||
cd .. | ||
pytest ${{ matrix.connector.name }} | ||
- name: Build ${{ matrix.connector.name }} Docker Image | ||
uses: docker/build-push-action@v2 | ||
with: | ||
context: . | ||
file: estuary-cdk/common.Dockerfile | ||
load: true | ||
build-args: | | ||
CONNECTOR_NAME=${{ matrix.connector.name }} | ||
CONNECTOR_TYPE=${{ matrix.connector.type }} | ||
tags: ghcr.io/estuary/${{ matrix.connector }}:local | ||
|
||
- name: Deployment | ||
uses: ./.github/actions/deploy | ||
with: | ||
connector: ${{ matrix.connector.name }} | ||
pg_database: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_DATABASE }} | ||
pg_host: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_HOST }} | ||
pg_password: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_PASSWORD }} | ||
pg_user: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_USER }} | ||
tag_sha: ${{ steps.setup.outputs.tag_sha }} | ||
tag_version: ${{ matrix.connector.version }} | ||
variants: ${{ matrix.connector.variants }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
# syntax=docker/dockerfile:1 | ||
FROM python:3.12-slim as base | ||
FROM base as builder | ||
|
||
ARG CONNECTOR_NAME | ||
|
||
RUN apt-get update && \ | ||
apt install -y --no-install-recommends \ | ||
python3-poetry | ||
|
||
RUN python -m venv /opt/venv | ||
ENV VIRTUAL_ENV=/opt/venv | ||
|
||
WORKDIR /opt/${CONNECTOR_NAME} | ||
COPY ${CONNECTOR_NAME} /opt/${CONNECTOR_NAME} | ||
COPY estuary-cdk /opt/estuary-cdk | ||
|
||
RUN poetry install | ||
|
||
|
||
FROM base as runner | ||
|
||
ARG CONNECTOR_NAME | ||
ARG CONNECTOR_TYPE | ||
ARG DOCS_URL | ||
|
||
LABEL FLOW_RUNTIME_PROTOCOL=${CONNECTOR_TYPE} | ||
LABEL FLOW_RUNTIME_CODEC=json | ||
|
||
COPY --from=builder /opt/$CONNECTOR_NAME /opt/$CONNECTOR_NAME | ||
COPY --from=builder /opt/estuary-cdk /opt/estuary-cdk | ||
COPY --from=builder /opt/venv /opt/venv | ||
|
||
ENV DOCS_URL=${DOCS_URL} | ||
ENV CONNECTOR_NAME=${CONNECTOR_NAME} | ||
|
||
CMD /opt/venv/bin/python -m $(echo "$CONNECTOR_NAME" | tr '-' '_') |