From 155b950435ef38f2e909ec2814f7b6e241f531ff Mon Sep 17 00:00:00 2001
From: Johnny Graettinger <johnny@estuary.dev>
Date: Thu, 29 Feb 2024 23:19:22 +0000
Subject: [PATCH] introduce new github workflow for python connectors

Begin to factor out common setup steps into reuseable composite actions.

Add a common estuary-cdk Dockerfile
---
 .github/actions/deploy/action.yaml | 78 +++++++++++++++++++++++++++
 .github/actions/setup/action.yaml  | 56 +++++++++++++++++++
 .github/workflows/ci.yaml          |  7 +--
 .github/workflows/python-new.yaml  | 86 ++++++++++++++++++++++++++++++
 estuary-cdk/common.Dockerfile      | 37 +++++++++++++
 5 files changed, 259 insertions(+), 5 deletions(-)
 create mode 100644 .github/actions/deploy/action.yaml
 create mode 100644 .github/actions/setup/action.yaml
 create mode 100644 .github/workflows/python-new.yaml
 create mode 100644 estuary-cdk/common.Dockerfile

diff --git a/.github/actions/deploy/action.yaml b/.github/actions/deploy/action.yaml
new file mode 100644
index 0000000000..6d6d6d0fc3
--- /dev/null
+++ b/.github/actions/deploy/action.yaml
@@ -0,0 +1,78 @@
+name: Common Deployment Steps
+description: Implements common deployment steps of all connector workflows
+inputs:
+  connector:
+    description: Image name of this connector.
+    required: true
+  pg_database:
+    description: Postgres database to use.
+    required: true
+  pg_host:
+    description: Postgres host to use.
+    required: true
+  pg_password:
+    description: Postgres password to use.
+    required: true
+  pg_user:
+    description: Postgres user to use.
+    required: true
+  tag_sha:
+    description: Short tag of the commit SHA1, such as 'f32dc1a'.
+    required: true
+  tag_version:
+    description: Version tag of the connector, such as `v1`.
+    required: true
+  variants:
+    description: Space-separated variant image names of this connector.
+    default: ''
+    required: false
+
+runs:
+  using: "composite"
+  steps:
+
+    - name: Push ${{ inputs.connector }} ${{ inputs.variants }} image(s) with commit SHA tag
+      shell: bash
+      run: |
+        for VARIANT in ${{ inputs.connector }} ${{ inputs.variants }}; do
+          echo "Building and pushing ${VARIANT}:${{ inputs.tag }}...";
+          docker build --build-arg BASE_CONNECTOR=ghcr.io/estuary/${{ inputs.connector }}:local \
+                       --build-arg DOCS_URL=https://go.estuary.dev/${VARIANT} \
+                       --tag ghcr.io/estuary/${VARIANT}:${{ inputs.tag_sha }} \
+                       --file connector-variant.Dockerfile .;
+          docker image push ghcr.io/estuary/${VARIANT}:${{ inputs.tag_sha }};
+        done
+
+    - name: Push ${{ inputs.connector }} image(s) with 'dev' and '${{ inputs.tag_version }}' tags
+      if: ${{ github.event_name == 'push' }}
+      shell: bash
+      run: |
+        for VARIANT in ${{ inputs.connector }} ${{ inputs.variants }}; do
+          docker image tag ghcr.io/estuary/${VARIANT}:${{ inputs.tag_sha }} ghcr.io/estuary/${VARIANT}:dev;
+          docker image tag ghcr.io/estuary/${VARIANT}:${{ inputs.tag_sha }} ghcr.io/estuary/${VARIANT}:${{ inputs.tag_version }};
+
+          docker image push ghcr.io/estuary/${VARIANT}:dev;
+          docker image push ghcr.io/estuary/${VARIANT}:${{ inputs.tag_version }};
+        done
+
+    - name: Install psql
+      if: ${{ github.event_name == 'push' }}
+      shell: bash
+      run: sudo apt install postgresql
+
+    - name: Refresh connector tags for ${{ inputs.connector }}
+      if: ${{ github.event_name == 'push' }}
+      shell: bash
+      env:
+        PGDATABASE: ${{ inputs.pg_database }}
+        PGHOST: ${{ inputs.pg_host }}
+        PGPASSWORD: ${{ inputs.pg_password }}
+        PGUSER: ${{ inputs.pg_user }}
+      run: |
+        for VARIANT in ${{ inputs.connector }} ${{ inputs.variants }}; do
+          echo "UPDATE connector_tags SET job_status='{\"type\": \"queued\"}'
+                  WHERE connector_id IN (
+                    SELECT id FROM connectors WHERE image_name='ghcr.io/estuary/${VARIANT}'
+                  ) AND image_tag IN (':${{ inputs.tag_version }}', ':dev');" | psql;
+        done
+
diff --git a/.github/actions/setup/action.yaml b/.github/actions/setup/action.yaml
new file mode 100644
index 0000000000..9cf29536fc
--- /dev/null
+++ b/.github/actions/setup/action.yaml
@@ -0,0 +1,56 @@
+name: Common Setup Steps
+description: Implements common setup steps of all connector workflows
+
+inputs:
+  github_token:
+    description: GitHub token to use.
+    required: true
+  gcp_project_id:
+    description: GCP Project for which to set up a service account.
+  gcp_service_account:
+    description: GCP Service account.
+
+outputs:
+  tag_sha:
+    description: Short tag of the commit SHA1
+    value: ${{ steps.determine-sha-tag.outputs.tag }}
+
+runs:
+  using: "composite"
+  steps:
+
+    - name: Determine current SHA tag.
+      shell: bash
+      id: determine-sha-tag
+      run: |
+        TAG=$(echo $GITHUB_SHA | head -c7)
+        echo ::set-output name=tag::${TAG}
+
+    - name: Download latest Flow release binaries and add them to $PATH
+      shell: bash
+      run: |
+        ./fetch-flow.sh
+        echo "${PWD}/flow-bin" >> $GITHUB_PATH
+    
+    - name: Login to GitHub package docker registry
+      shell: bash
+      run: |
+        echo "${{ inputs.github_token }}" | \
+          docker login --username ${{ github.actor }} --password-stdin ghcr.io
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+      with:
+        driver-opts: network=host
+
+    - name: Create docker network flow-test
+      shell: bash
+      run: docker network create flow-test
+
+    - name: Set up Cloud SDK
+      if: ${{ inputs.gcp_project_id }}
+      uses: google-github-actions/setup-gcloud@v0
+      with:
+        project_id: ${{ inputs.gcp_project_id }}
+        service_account_key: ${{ inputs.gcp_service_account }}
+        export_default_credentials: true
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 2c608ba1ab..be86f5a7a9 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -7,9 +7,9 @@ concurrency:
 on:
   push:
     # TODO: Remove jshearer/python_connector_testing before merging
-    branches: [main, jshearer/python_connector_testing]
+    branches: [nothing]
   pull_request:
-    branches: [main]
+    branches: [nothing]
 
 jobs:
   build_base_image:
@@ -121,9 +121,6 @@ jobs:
           - connector: source-shopify
             connector_type: capture
             python: true
-          - connector: source-asana
-            connector_type: capture
-            python: true
           - connector: source-airtable
             connector_type: capture
             python: true
diff --git a/.github/workflows/python-new.yaml b/.github/workflows/python-new.yaml
new file mode 100644
index 0000000000..60a38ade38
--- /dev/null
+++ b/.github/workflows/python-new.yaml
@@ -0,0 +1,86 @@
+name: Python-Connectors
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "estuary-cdk/**"
+      - "source-asana/**"
+      - "source-google-sheets-native/**"
+      - "source-hubspot-native/**"
+  pull_request:
+    branches: [main]
+    paths:
+      - "estuary-cdk/**"
+      - "source-asana/**"
+      - "source-google-sheets-native/**"
+      - "source-hubspot-native/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  py_connector:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        connector:
+          - name: source-asana
+            type: capture
+            version: v1
+          - name: source-hubspot-native
+            type: capture
+            version: v1
+          - name: source-google-sheets-native
+            type: capture
+            version: v1
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup
+        id: setup
+        uses: .github/actions/setup
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+
+      - name: ${{ matrix.connector.name }} tests
+        run: |
+          cd ${{ matrix.connector.name }}
+          poetry install
+          source $(poetry env info --path)/bin/activate
+          cd ..
+          pytest .
+
+      - name: Build ${{ matrix.connector.name }} Docker Image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: estuary-cdk/common.Dockerfile
+          load: true
+          build-args: |
+            CONNECTOR_NAME=${{ matrix.connector.name }}
+            CONNECTOR_TYPE=${{ matrix.connector.type }}
+          tags: ghcr.io/estuary/${{ matrix.connector }}:local
+
+      - id: Deployment
+        uses: .github/actions/deploy
+        with:
+          connector: ${{ matrix.connector.name }}
+          pg_database: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_DATABASE }}
+          pg_host: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_HOST }}
+          pg_password: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_PASSWORD }}
+          pg_user: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_USER }}
+          tag_sha: ${{ steps.setup.outputs.tag_sha }}
+          tag_version: ${{ matrix.connector.version }}
+          variants: ${{ matrix.connector.variants }}
\ No newline at end of file
diff --git a/estuary-cdk/common.Dockerfile b/estuary-cdk/common.Dockerfile
new file mode 100644
index 0000000000..c51e5b28de
--- /dev/null
+++ b/estuary-cdk/common.Dockerfile
@@ -0,0 +1,37 @@
+# syntax=docker/dockerfile:1
+FROM python:3.12-slim as base
+FROM base as builder
+
+ARG CONNECTOR_NAME
+
+RUN apt-get update && \
+    apt install -y --no-install-recommends \
+    python3-poetry 
+
+RUN python -m venv /opt/venv
+ENV VIRTUAL_ENV=/opt/venv
+
+WORKDIR /opt/${CONNECTOR_NAME}
+COPY ${CONNECTOR_NAME} /opt/${CONNECTOR_NAME}
+COPY estuary-cdk /opt/estuary-cdk
+
+RUN poetry install
+
+
+FROM base as runner
+
+ARG CONNECTOR_NAME
+ARG CONNECTOR_TYPE
+ARG DOCS_URL
+
+LABEL FLOW_RUNTIME_PROTOCOL=${CONNECTOR_TYPE}
+LABEL FLOW_RUNTIME_CODEC=json
+
+COPY --from=builder /opt/$CONNECTOR_NAME /opt/$CONNECTOR_NAME
+COPY --from=builder /opt/estuary-cdk /opt/estuary-cdk
+COPY --from=builder /opt/venv /opt/venv
+
+ENV DOCS_URL=${DOCS_URL}
+ENV CONNECTOR_NAME=${CONNECTOR_NAME}
+
+CMD /opt/venv/bin/python -m $(echo "$CONNECTOR_NAME" | tr '-' '_')
\ No newline at end of file