Skip to content

Commit

Permalink
introduce new github workflow for python connectors
Browse files Browse the repository at this point in the history
Begin to factor out common setup steps into reuseable composite actions.

Add a common estuary-cdk Dockerfile
  • Loading branch information
jgraettinger committed Feb 29, 2024
1 parent 951a0ba commit 1360a60
Show file tree
Hide file tree
Showing 5 changed files with 259 additions and 5 deletions.
78 changes: 78 additions & 0 deletions .github/actions/deploy/action.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
name: Common Deployment Steps
description: Implements common deployment steps of all connector workflows
inputs:
connector:
description: Image name of this connector.
required: true
pg_database:
description: Postgres database to use.
required: true
pg_host:
description: Postgres host to use.
required: true
pg_password:
description: Postgres password to use.
required: true
pg_user:
description: Postgres user to use.
required: true
tag_sha:
description: Short tag of the commit SHA1, such as 'f32dc1a'.
required: true
tag_version:
description: Version tag of the connector, such as `v1`.
required: true
variants:
description: Space-separated variant image names of this connector.
default: ''
required: false

runs:
using: "composite"
steps:

- name: Push ${{ inputs.connector }} ${{ inputs.variants }} image(s) with commit SHA tag
shell: bash
run: |
for VARIANT in ${{ inputs.connector }} ${{ inputs.variants }}; do
echo "Building and pushing ${VARIANT}:${{ inputs.tag }}...";
docker build --build-arg BASE_CONNECTOR=ghcr.io/estuary/${{ inputs.connector }}:local \
--build-arg DOCS_URL=https://go.estuary.dev/${VARIANT} \
--tag ghcr.io/estuary/${VARIANT}:${{ inputs.tag_sha }} \
--file connector-variant.Dockerfile .;
docker image push ghcr.io/estuary/${VARIANT}:${{ inputs.tag_sha }};
done
- name: Push ${{ inputs.connector }} image(s) with 'dev' and '${{ inputs.tag_version }}' tags
if: ${{ github.event_name == 'push' }}
shell: bash
run: |
for VARIANT in ${{ inputs.connector }} ${{ inputs.variants }}; do
docker image tag ghcr.io/estuary/${VARIANT}:${{ inputs.tag_sha }} ghcr.io/estuary/${VARIANT}:dev;
docker image tag ghcr.io/estuary/${VARIANT}:${{ inputs.tag_sha }} ghcr.io/estuary/${VARIANT}:${{ inputs.tag_version }};
docker image push ghcr.io/estuary/${VARIANT}:dev;
docker image push ghcr.io/estuary/${VARIANT}:${{ inputs.tag_version }};
done
- name: Install psql
if: ${{ github.event_name == 'push' }}
shell: bash
run: sudo apt install postgresql

- name: Refresh connector tags for ${{ inputs.connector }}
if: ${{ github.event_name == 'push' }}
shell: bash
env:
PGDATABASE: ${{ inputs.pg_database }}
PGHOST: ${{ inputs.pg_host }}
PGPASSWORD: ${{ inputs.pg_password }}
PGUSER: ${{ inputs.pg_user }}
run: |
for VARIANT in ${{ inputs.connector }} ${{ inputs.variants }}; do
echo "UPDATE connector_tags SET job_status='{\"type\": \"queued\"}'
WHERE connector_id IN (
SELECT id FROM connectors WHERE image_name='ghcr.io/estuary/${VARIANT}'
) AND image_tag IN (':${{ inputs.tag_version }}', ':dev');" | psql;
done
56 changes: 56 additions & 0 deletions .github/actions/setup/action.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
name: Common Setup Steps
description: Implements common setup steps of all connector workflows

inputs:
github_token:
description: GitHub token to use.
required: true
gcp_project_id:
description: GCP Project for which to set up a service account.
gcp_service_account:
description: GCP Service account.

outputs:
tag_sha:
description: Short tag of the commit SHA1
value: ${{ steps.determine-sha-tag.outputs.tag }}

runs:
using: "composite"
steps:

- name: Determine current SHA tag.
shell: bash
id: determine-sha-tag
run: |
TAG=$(echo $GITHUB_SHA | head -c7)
echo "tag=${TAG}" >> $GITHUB_OUTPUT
- name: Download latest Flow release binaries and add them to $PATH
shell: bash
run: |
./fetch-flow.sh
echo "${PWD}/flow-bin" >> $GITHUB_PATH
- name: Login to GitHub package docker registry
shell: bash
run: |
echo "${{ inputs.github_token }}" | \
docker login --username ${{ github.actor }} --password-stdin ghcr.io
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
with:
driver-opts: network=host

- name: Create docker network flow-test
shell: bash
run: docker network create flow-test

- name: Set up Cloud SDK
if: ${{ inputs.gcp_project_id }}
uses: google-github-actions/setup-gcloud@v0
with:
project_id: ${{ inputs.gcp_project_id }}
service_account_key: ${{ inputs.gcp_service_account }}
export_default_credentials: true
7 changes: 2 additions & 5 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ concurrency:
on:
push:
# TODO: Remove jshearer/python_connector_testing before merging
branches: [main, jshearer/python_connector_testing]
branches: [nothing]
pull_request:
branches: [main]
branches: [nothing]

jobs:
build_base_image:
Expand Down Expand Up @@ -121,9 +121,6 @@ jobs:
- connector: source-shopify
connector_type: capture
python: true
- connector: source-asana
connector_type: capture
python: true
- connector: source-airtable
connector_type: capture
python: true
Expand Down
86 changes: 86 additions & 0 deletions .github/workflows/python-new.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
name: Python-Connectors

on:
push:
branches: [main]
paths:
- "estuary-cdk/**"
- "source-asana/**"
- "source-google-sheets-native/**"
- "source-hubspot-native/**"
pull_request:
branches: [main]
paths:
- "estuary-cdk/**"
- "source-asana/**"
- "source-google-sheets-native/**"
- "source-hubspot-native/**"

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

jobs:
py_connector:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
connector:
- name: source-asana
type: capture
version: v1
- name: source-hubspot-native
type: capture
version: v1
- name: source-google-sheets-native
type: capture
version: v1

steps:
- uses: actions/checkout@v4

- name: Common Setup
id: setup
uses: ./.github/actions/setup
with:
github_token: ${{ secrets.GITHUB_TOKEN }}

- name: Install Python
uses: actions/setup-python@v5
with:
python-version: "3.12"

- name: Install Poetry
uses: snok/install-poetry@v1

- name: ${{ matrix.connector.name }} tests
run: |
cd ${{ matrix.connector.name }}
poetry install
source $(poetry env info --path)/bin/activate
cd ..
pytest ${{ matrix.connector.name }}
- name: Build ${{ matrix.connector.name }} Docker Image
uses: docker/build-push-action@v2
with:
context: .
file: estuary-cdk/common.Dockerfile
load: true
build-args: |
CONNECTOR_NAME=${{ matrix.connector.name }}
CONNECTOR_TYPE=${{ matrix.connector.type }}
tags: ghcr.io/estuary/${{ matrix.connector }}:local

- name: Deployment
uses: ./.github/actions/deploy
with:
connector: ${{ matrix.connector.name }}
pg_database: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_DATABASE }}
pg_host: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_HOST }}
pg_password: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_PASSWORD }}
pg_user: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_USER }}
tag_sha: ${{ steps.setup.outputs.tag_sha }}
tag_version: ${{ matrix.connector.version }}
variants: ${{ matrix.connector.variants }}
37 changes: 37 additions & 0 deletions estuary-cdk/common.Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# syntax=docker/dockerfile:1
FROM python:3.12-slim as base
FROM base as builder

ARG CONNECTOR_NAME

RUN apt-get update && \
apt install -y --no-install-recommends \
python3-poetry

RUN python -m venv /opt/venv
ENV VIRTUAL_ENV=/opt/venv

WORKDIR /opt/${CONNECTOR_NAME}
COPY ${CONNECTOR_NAME} /opt/${CONNECTOR_NAME}
COPY estuary-cdk /opt/estuary-cdk

RUN poetry install


FROM base as runner

ARG CONNECTOR_NAME
ARG CONNECTOR_TYPE
ARG DOCS_URL

LABEL FLOW_RUNTIME_PROTOCOL=${CONNECTOR_TYPE}
LABEL FLOW_RUNTIME_CODEC=json

COPY --from=builder /opt/$CONNECTOR_NAME /opt/$CONNECTOR_NAME
COPY --from=builder /opt/estuary-cdk /opt/estuary-cdk
COPY --from=builder /opt/venv /opt/venv

ENV DOCS_URL=${DOCS_URL}
ENV CONNECTOR_NAME=${CONNECTOR_NAME}

CMD /opt/venv/bin/python -m $(echo "$CONNECTOR_NAME" | tr '-' '_')

0 comments on commit 1360a60

Please sign in to comment.