Notion schema fix and Cron schedule change (#2793) #1230
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Publish workflows-service image | |
| on: | |
| repository_dispatch: | |
| types: [invoke-cd-from-external-repo] | |
| workflow_dispatch: | |
| inputs: | |
| deploy_to_dev: | |
| type: choice | |
| description: 'Deploy to Development Environment' | |
| required: true | |
| default: 'false' | |
| options: | |
| - 'false' | |
| - 'true' | |
| push: | |
| paths: | |
| # Run this pipeline only if there are changes in specified path | |
| - 'services/workflows-service/**' | |
| branches: | |
| - dev | |
| - test | |
| - prod | |
| - staging | |
| - sb | |
| - demo | |
| env: | |
| REGISTRY: ghcr.io | |
| IMAGE_NAME: ${{ github.repository_owner }}/workflows-service | |
| jobs: | |
| determine_branch: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| branch: ${{ steps.set-branch.outputs.branch }} | |
| steps: | |
| - name: Set branch | |
| id: set-branch | |
| run: | | |
| if [ "${{ github.event_name }}" == "repository_dispatch" ]; then | |
| echo "branch=${{ github.event.client_payload.ref }}" >> $GITHUB_OUTPUT | |
| else | |
| echo "branch=${{ github.ref_name }}" >> $GITHUB_OUTPUT | |
| fi | |
| echo "local branch=${{ github.ref_name }}, client_payload ref=${{ github.event.client_payload.ref }}" | |
| build-and-push-image: | |
| runs-on: ubuntu-latest | |
| needs: [determine_branch] | |
| permissions: | |
| contents: read | |
| packages: write | |
| outputs: | |
| sha_short: ${{ steps.version.outputs.sha_short }} # short sha of the commit | |
| image_tags: ${{ steps.docker_meta.outputs.tags }} # <short_sha>-<branch_name>, <branch_name>, latest(for prod branch only) | |
| version: ${{ steps.bump-version.outputs.version }} # workflow-service@vX.X.X | |
| bumped_tag: ${{ steps.bump-version.outputs.tag }} # bumped patched version X.X.X+1 | |
| docker_image: ${{ steps.docker-version.outputs.image }} # ghcr.io/ballerine-io/workflows-service | |
| docker_tag: ${{ steps.docker-version.outputs.tag }} # <short_sha>-<branch_name> | |
| docker_full_image: ${{ steps.docker-version.outputs.full_image }} # ghcr.io/ballerine-io/workflows-service:<short_sha>-<branch_name> | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Get tags | |
| run: git fetch --tags origin | |
| - name: Get version | |
| id: version | |
| run: | | |
| TAG=$(git tag -l "$(echo workflow-service@)*" | sort -V -r | head -n 1) | |
| echo "tag=$TAG" | |
| echo "tag=$TAG" >> "$GITHUB_OUTPUT" | |
| echo "TAG=$TAG" >> "$GITHUB_ENV" | |
| SHORT_SHA=$(git rev-parse --short HEAD) | |
| echo "sha_short=$SHORT_SHA" | |
| echo "sha_short=$SHORT_SHA" >> $GITHUB_OUTPUT | |
| echo "SHORT_SHA=$SHORT_SHA" >> $GITHUB_ENV | |
| - name: Bump version | |
| id: bump-version | |
| uses: ./.github/actions/bump-version | |
| with: | |
| tag: ${{ steps.version.outputs.tag }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| with: | |
| platforms: 'arm64,arm' | |
| - name: Cache Docker layers | |
| id: cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: /tmp/.buildx-cache | |
| key: ${{ runner.os }}-docker-${{ hashFiles('**/Dockerfile') }} | |
| restore-keys: | | |
| ${{ runner.os }}-docker-${{ hashFiles('**/Dockerfile') }} | |
| ${{ runner.os }}-docker- | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract metadata for Docker images | |
| id: docker_meta | |
| uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 | |
| with: | |
| images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} | |
| tags: | | |
| type=raw,value=${{ needs.determine_branch.outputs.branch }} | |
| type=raw,value=${{ steps.version.outputs.sha_short }}-${{ needs.determine_branch.outputs.branch }} | |
| type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'prod') }} | |
| type=sha,format=short | |
| - name: Docker metadata version | |
| id: docker-version | |
| run: | | |
| DOCKER_IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} | |
| DOCKER_TAG=${{ steps.version.outputs.sha_short }}-${{ needs.determine_branch.outputs.branch }} | |
| DOCKER_FULL_IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.version.outputs.sha_short }}-${{ needs.determine_branch.outputs.branch }} | |
| echo "DOCKER_IMAGE=$DOCKER_IMAGE" | |
| echo "DOCKER_TAG=$DOCKER_TAG" | |
| echo "DOCKER_FULL_IMAGE=$DOCKER_FULL_IMAGE" | |
| echo "image=$DOCKER_IMAGE" >> $GITHUB_OUTPUT | |
| echo "tag=$DOCKER_TAG" >> $GITHUB_OUTPUT | |
| echo "full_image=$DOCKER_FULL_IMAGE" >> $GITHUB_OUTPUT | |
| - name: Print docker version outputs | |
| run: | | |
| echo "Metadata: ${{ steps.docker_meta.outputs.tags }}" | |
| echo "sha_short: ${{ steps.version.outputs.sha_short }}" | |
| echo "docker_meta-tags: ${{ steps.docker_meta.outputs.tags }}" | |
| echo "bump-version-version: ${{ steps.bump-version.outputs.version }}" | |
| echo "bump-version-tag: ${{ steps.bump-version.outputs.tag }}" | |
| - name: Build and push Docker image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: services/workflows-service | |
| platforms: linux/amd64 | |
| push: true | |
| cache-from: type=local,src=/tmp/.buildx-cache | |
| cache-to: type=local,dest=/tmp/.buildx-cache,mode=max | |
| tags: ${{ steps.docker_meta.outputs.tags }} | |
| build-args: | | |
| "RELEASE=${{ steps.bump-version.outputs.tag }}" | |
| "SHORT_SHA=${{ steps.version.outputs.sha_short }}" | |
| - name: Scan Docker Image | |
| uses: aquasecurity/trivy-action@master | |
| continue-on-error: true | |
| with: | |
| cache-dir: | |
| image-ref: ${{ steps.docker-version.outputs.full_image }} | |
| format: 'table' | |
| ignore-unfixed: true | |
| exit-code: 1 | |
| trivyignores: ./.trivyignore | |
| vuln-type: 'os,library' | |
| severity: 'CRITICAL' | |
| update-helm-chart: | |
| needs: [determine_branch, build-and-push-ee-image] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| - name: Checkout cloud-infra-config repository | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ballerine-io/cloud-infra-config | |
| token: ${{ secrets.GIT_TOKEN }} | |
| ref: main | |
| fetch-depth: 1 | |
| sparse-checkout: | | |
| kubernetes/helm/wf-service | |
| sparse-checkout-cone-mode: true | |
| - name: Check if values yaml file exists | |
| id: update_helm_check | |
| shell: bash | |
| run: | | |
| if [ -f "kubernetes/helm/wf-service/${{ needs.determine_branch.outputs.branch }}-custom-values.yaml" ]; then | |
| echo "file_name=${{ needs.determine_branch.outputs.branch }}-custom-values.yaml" >> "$GITHUB_OUTPUT" | |
| elif [ "${{ github.event.inputs.deploy_to_dev }}" == "true" ]; then | |
| echo "file_name=dev-custom-values.yaml" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "skip_helm=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Update workdlow-service image version in the HelmChart | |
| if: ${{ steps.update_helm_check.outputs.skip_helm != 'true' }} | |
| uses: fjogeleit/yaml-update-action@main | |
| with: | |
| repository: ballerine-io/cloud-infra-config | |
| branch: main | |
| commitChange: true | |
| message: 'Update wf-service image Version to sha-${{ needs.build-and-push-ee-image.outputs.wf_m_sha_short }} - (Commit hash: ${{ github.sha }}, commit message: ${{ github.event.head_commit.message }})' | |
| token: ${{ secrets.GIT_TOKEN }} | |
| changes: | | |
| { | |
| "kubernetes/helm/wf-service/${{steps.update_helm_check.outputs.file_name}}": { | |
| "image.tag": "${{ needs.build-and-push-ee-image.outputs.docker_tag }}", | |
| "prismaMigrate.image.tag": "${{ needs.build-and-push-ee-image.outputs.docker_tag }}", | |
| "dbMigrate.image.tag": "${{ needs.build-and-push-ee-image.outputs.docker_tag }}", | |
| "dataSync.image.tag": "${{ needs.build-and-push-ee-image.outputs.docker_tag }}" | |
| } | |
| } | |
| release: | |
| runs-on: ubuntu-latest | |
| if: startsWith(github.ref, 'refs/heads/prod') || startsWith(github.ref, 'refs/heads/dev') || startsWith(github.ref, 'refs/heads/sb') || github.event.inputs.deploy_to_dev == 'true' | |
| needs: build-and-push-image | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Release | |
| run: | | |
| if [ "${{ github.event.inputs.deploy_to_dev }}" == "true" ]; then | |
| suffix="-dev-${{ needs.build-and-push-image.outputs.sha_short }}" | |
| else | |
| suffix="" | |
| fi | |
| gh release create ${{ needs.build-and-push-image.outputs.version }}${suffix} --notes-start-tag ${{ needs.build-and-push-image.outputs.bumped_tag }} | |
| sentry: | |
| runs-on: ubuntu-latest | |
| needs: [build-and-push-image, release] | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| # TODO: add caching for docker_full_image which build previously | |
| - name: Run Container and Copy File | |
| run: | | |
| id=$(docker run --rm --name tmp -d ${{ needs.build-and-push-image.outputs.docker_full_image }} tail -f /dev/null) | |
| mkdir -p ./dist | |
| docker cp $id:/app/dist/ ./dist | |
| curl -sL https://sentry.io/get-cli/ | SENTRY_CLI_VERSION="2.31.0" bash | |
| sentry-cli releases new "${{needs.build-and-push-image.outputs.version}}" | |
| echo "sentry-cli releases new ${{needs.build-and-push-image.outputs.version}}" | |
| sentry-cli releases set-commits "${{needs.build-and-push-image.outputs.version}}" --auto --ignore-missing | |
| echo "sentry-cli releases set-commits ${{needs.build-and-push-image.outputs.version}} --auto --ignore-missing" | |
| sentry-cli sourcemaps upload --dist="${{needs.build-and-push-image.outputs.sha_short}}" --release="${{needs.build-and-push-image.outputs.version}}" ./dist | |
| echo "sentry-cli sourcemaps upload --dist=${{needs.build-and-push-image.outputs.sha_short}} --release=${{needs.build-and-push-image.outputs.version}} ./dist" | |
| env: | |
| SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} | |
| SENTRY_ORG: ${{ secrets.SENTRY_ORG }} | |
| SENTRY_PROJECT: ${{ secrets.WF_SENTRY_PROJECT }} | |
| check_if_data_migration_needed: | |
| runs-on: ubuntu-latest | |
| needs: determine_branch | |
| outputs: | |
| should_build: ${{ steps.check-branch-existance.outputs.should_build }} # short sha of the commit | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| submodules: 'recursive' | |
| token: ${{ secrets.SUBMODULES_TOKEN }} | |
| - name: Check if branch exists | |
| id: check-branch-existance | |
| run: | | |
| cd services/workflows-service/prisma/data-migrations | |
| git fetch --no-tags --depth=1 origin +refs/heads/dev:refs/remotes/origin/${{ needs.determine_branch.outputs.branch }} | |
| git checkout -b ${{ needs.determine_branch.outputs.branch }} origin/${{ needs.determine_branch.outputs.branch }} | |
| git config --global user.email "github-actions@example.com" | |
| git config --global user.name "GitHub Actions" | |
| git reset --hard HEAD | |
| git pull origin ${{ needs.determine_branch.outputs.branch }} --rebase | |
| is_exists=$(git ls-remote --exit-code --heads -t --ref -q origin "${{ needs.determine_branch.outputs.branch }}" | wc -l) | |
| # Check if the branch exists by counting the number of results | |
| if [ $is_exists -eq 0 ]; then | |
| echo "Branch '${{ needs.determine_branch.outputs.branch }}' does not exist." | |
| echo "should_build=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "should_build=true" >> $GITHUB_OUTPUT | |
| fi | |
| exit 0 | |
| build-and-push-ee-image: | |
| runs-on: ubuntu-latest | |
| needs: [build-and-push-image, check_if_data_migration_needed, determine_branch] | |
| if: ${{ needs.check_if_data_migration_needed.outputs.should_build == 'true' }} | |
| outputs: | |
| wf_m_sha_short: ${{ steps.fetch-submodule.outputs.wf_m_sha_short }} # short sha of the commit | |
| docker_tag: ${{ steps.docker-version.outputs.wf_m_tag }} | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| submodules: 'recursive' | |
| token: ${{ secrets.SUBMODULES_TOKEN }} | |
| - name: Cache Docker layers | |
| id: cache | |
| uses: actions/cache@v2 | |
| with: | |
| path: /tmp/.buildx-cache | |
| key: ${{ runner.os }}-docker-${{ hashFiles('**/Dockerfile') }} | |
| restore-keys: | | |
| ${{ runner.os }}-docker-${{ hashFiles('**/Dockerfile') }} | |
| ${{ runner.os }}-docker- | |
| - name: Fetch submodule branch | |
| id: fetch-submodule | |
| run: | | |
| cd services/workflows-service/prisma/data-migrations | |
| git fetch --no-tags --depth=1 origin +refs/heads/dev:refs/remotes/origin/${{ needs.determine_branch.outputs.branch }} | |
| git checkout -b ${{ needs.determine_branch.outputs.branch }} origin/${{ needs.determine_branch.outputs.branch }} | |
| git config --global user.email "github-actions@example.com" | |
| git config --global user.name "GitHub Actions" | |
| git reset --hard HEAD | |
| git pull origin ${{ needs.determine_branch.outputs.branch }} --rebase | |
| WF_M_SHORT_SHA=$(git rev-parse --short HEAD) | |
| echo "sha_short=$WF_M_SHORT_SHA" | |
| echo "wf_m_sha_short=$WF_M_SHORT_SHA" >> $GITHUB_OUTPUT | |
| cd ../../../.. | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| with: | |
| platforms: 'arm64,arm' | |
| - name: Log in to the container registry | |
| uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract metadata for ee Docker images | |
| id: eemeta | |
| uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 | |
| with: | |
| images: ${{needs.build-and-push-image.outputs.docker_image}}-ee | |
| tags: | | |
| type=raw,value=${{ needs.determine_branch.outputs.branch }} | |
| type=raw,value=${{ needs.build-and-push-image.outputs.sha_short }}-${{ steps.fetch-submodule.outputs.wf_m_sha_short }}-${{ needs.determine_branch.outputs.branch }} | |
| type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'prod') }} | |
| type=sha,format=short | |
| - name: Docker metadata version | |
| id: docker-version | |
| run: | | |
| DOCKER_IMAGE=${{needs.build-and-push-image.outputs.docker_image}}-ee | |
| DOCKER_TAG=${{ needs.build-and-push-image.outputs.sha_short }}-${{ steps.fetch-submodule.outputs.wf_m_sha_short }}-${{ needs.determine_branch.outputs.branch }} | |
| DOCKER_FULL_IMAGE=$DOCKER_IMAGE:$DOCKER_TAG | |
| echo "DOCKER_IMAGE=$DOCKER_IMAGE" | |
| echo "DOCKER_TAG=$DOCKER_TAG" | |
| echo "DOCKER_FULL_IMAGE=$DOCKER_FULL_IMAGE" | |
| echo "wf_m_image=$DOCKER_IMAGE" >> $GITHUB_OUTPUT | |
| echo "wf_m_tag=$DOCKER_TAG" >> $GITHUB_OUTPUT | |
| echo "wf_m_full_image=$DOCKER_FULL_IMAGE" >> $GITHUB_OUTPUT | |
| - name: Build and push ee Docker image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: services/workflows-service | |
| file: services/workflows-service/Dockerfile.ee | |
| platforms: linux/amd64 | |
| push: true | |
| cache-from: type=local,src=/tmp/.buildx-cache | |
| tags: ${{ steps.eemeta.outputs.tags }} | |
| build-args: | | |
| "BASE_IMAGE=${{needs.build-and-push-image.outputs.docker_full_image}}" | |
| "RELEASE=${{ needs.build-and-push-image.outputs.bumped_tag }}" | |
| "SHORT_SHA=${{ needs.build-and-push-image.outputs.sha_short }}" |