CI.yaml

name: CI

on:
  push:
    branches:
      - main
    paths-ignore: ["docs/**", "scripts/**", "**.md", "*"]
  pull_request:
    branches:
      - main
    types: [opened, synchronize, labeled, reopened, ready_for_review]
    paths-ignore: ["docs/**", "scripts/**", "**.md", "*"]
  
  workflow_dispatch:
    inputs:
      debug_enabled:
        description: "Run with tmate.io debugging enabled"
        required: true
        type: boolean
        default: false
      run_build_images:
        description: "Run build-images job"
        required: false
        type: boolean
        default: false
      run_execute_workflow:
        description: "Run execute-workflow job"
        required: false
        type: boolean
        default: false
      workflow_execution_mode:
        description: "Workflow execution mode"
        required: false
        type: string
        default: "prod"

defaults:
  run:
    shell: bash

permissions:
  actions: write
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:

  set-variables:
    name: Set Variables
    runs-on: ubuntu-latest
    outputs:
      debug: ${{ steps.set-variables.outputs.debug }}
      skip_ci: ${{ steps.set-variables.outputs.skip_ci }}
      skip_tests: ${{ steps.set-variables.outputs.skip_tests }}
      mode: ${{ steps.set-variables.outputs.mode }}
  
    steps:
    - name: Set action variables
      id: set-variables
      run: |
        DEBUG="false"
        MODE="prod"
        SKIP_CI="false"
        SKIP_TESTS="false"

        if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
          DEBUG="${{ github.event.inputs.debug_enabled }}"
          MODE="${{ github.event.inputs.workflow_execution_mode }}"
        fi

        if [[ "${{ github.event_name }}" == "pull_request" ]]; then
          if ${{ contains(github.event.pull_request.labels.*.name, 'skip-ci') }}; then
            SKIP_CI="true"
          fi
          if ${{ contains(github.event.pull_request.labels.*.name, 'skip-tests') }}; then
            SKIP_TESTS="true"
          fi
          if ${{ contains(github.event.pull_request.labels.*.name, 'actions-debug') }}; then
            DEBUG="true"
          fi
          if ${{ contains(github.event.pull_request.labels.*.name, 'workflow-mode-dev') }}; then
            MODE="dev"
          fi
        fi

        echo "DEBUG=$DEBUG"
        echo "MODE=$MODE"
        echo "SKIP_CI=$SKIP_CI"
        echo "SKIP_TESTS=$SKIP_TESTS"

        echo "DEBUG=$DEBUG" >> $GITHUB_OUTPUT
        echo "MODE=$MODE" >> $GITHUB_OUTPUT
        echo "SKIP_CI=$SKIP_CI" >> $GITHUB_OUTPUT
        echo "SKIP_TESTS=$SKIP_TESTS" >> $GITHUB_OUTPUT


  test:
    runs-on: ubuntu-latest
    needs: [set-variables]
    if: ${{ needs.set-variables.outputs.skip_ci != 'true' && needs.set-variables.outputs.skip_tests != 'true' }} 
    strategy:
      matrix:
        python_version: ['3.10']

    steps:
    - name: Check Variables
      run: |
        echo "SKIP_CI=${{ needs.set-variables.outputs.skip_ci }}"
        echo "SKIP_TESTS=${{ needs.set-variables.outputs.skip_tests }}"
        echo "DEBUG=${{ needs.set-variables.outputs.debug }}"
        echo "MODE=${{ needs.set-variables.outputs.mode }}"
    - name: Checkout code
      uses: actions/checkout@v4

    - name: Setup environment
      uses: ./.github/actions/setup_environment
      with:
        python_version: ${{ matrix.python_version }}
        debug_enabled: ${{ needs.set-variables.outputs.debug }}

    - name: "Setup tmate debug session"
      uses: mxschmitt/action-tmate@v3
      if: ${{ inputs.debug_enabled }}

    - name: Lint and typecheck
      run: |
        make lint-check

    - name: Run tests
      run: |
        make test-cov-xml

    - name: Upload coverage
      uses: codecov/codecov-action@v3
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        fail_ci_if_error: false
        verbose: true


  build-images:
    needs: [test, set-variables]
    if: ${{ needs.set-variables.outputs.skip_ci != 'true' && (contains(github.event.pull_request.labels.*.name, 'build-images') || contains(github.event.pull_request.labels.*.name, 'execute-workflow') || (github.event_name == 'workflow_dispatch' && github.event.inputs.run_build_images)) }}
    uses: ./.github/workflows/build-images.yaml
    secrets:
      GCP_ARTIFACT_REGISTRY_PATH: ${{ secrets.GCP_ARTIFACT_REGISTRY_PATH }}
    with:
      debug_enabled: ${{ needs.set-variables.outputs.debug }}
      mode: ${{ needs.set-variables.outputs.mode }}


  config-workflows:
    needs: [test, set-variables]
    if: ${{ needs.set-variables.outputs.skip_ci != 'true' && (contains(github.event.pull_request.labels.*.name, 'execute-workflow') || (github.event_name == 'workflow_dispatch' && github.event.inputs.run_execute_workflow)) }}
    runs-on: ubuntu-latest
    outputs:
      config-path: ${{ steps.config-output.outputs.path }}

    steps:
    - name: Checkout code
      uses: actions/checkout@v4

    - name: Create Flyte config from YAML template
      id: yq-process
      uses: mikefarah/yq@master
      with:
        cmd: "yq e \
              '.admin.endpoint = strenv(FLYTE_CLUSTER_ENDPOINT) | \
              .storage.stow.config.project_id = strenv(GCP_PROJECT_ID) | \
              .storage.stow.config.scopes = strenv(GCP_STORAGE_SCOPES) | \
              .storage.container = strenv(GCP_STORAGE_CONTAINER)' \
              .flyte/config-template.yaml > .flyte/config.yaml"
      env:
        FLYTE_CLUSTER_ENDPOINT: ${{ secrets.FLYTE_CLUSTER_ENDPOINT }}
        GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
        GCP_STORAGE_SCOPES: ${{ secrets.GCP_STORAGE_SCOPES }}
        GCP_STORAGE_CONTAINER: ${{ secrets.GCP_STORAGE_CONTAINER }}

    - name: Upload Flyte config as an artifact
      id: config-output
      uses: actions/upload-artifact@v3
      with:
        name: flyte-config
        path: ${{ secrets.FLYTECTL_CONFIG }}


  execute-workflow:
    needs: [config-workflows, build-images, set-variables]
    if: ${{ needs.set-variables.outputs.skip_ci != 'true' && (contains(github.event.pull_request.labels.*.name, 'execute-workflow') || (github.event_name == 'workflow_dispatch' && github.event.inputs.run_execute_workflow)) }}
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python_version: ['3.10']
    env:
        FLYTECTL_CONFIG: ${{ secrets.FLYTECTL_CONFIG }}
        FLYTE_OAUTH_CLIENT_SECRET: ${{ secrets.FLYTE_OAUTH_CLIENT_SECRET }}
        WORKFLOW_IMAGE: ${{ vars.WORKFLOW_IMAGE }}

    steps:
    - name: Checkout code
      uses: actions/checkout@v4
  
    - name: Inject slug/short variables
      uses: rlespinasse/github-slug-action@v4

    - name: Setup environment
      uses: ./.github/actions/setup_environment
      with:
        python_version: ${{ matrix.python_version }}
        debug_enabled: ${{ needs.set-variables.outputs.debug }}

    - name: Download Flyte config
      uses: actions/download-artifact@v3
      with:
        name: flyte-config
        path: .flyte/

    - name: Setup tmate debug session
      if: ${{ github.event.inputs.debug_enabled == 'true' }}
      uses: mxschmitt/action-tmate@v3
  
    - name: Execute workflow
      id: execute
      run: |
        make run_${{ needs.set-variables.outputs.mode }}

    - name: Create config tarball
      id: save-hydra-outputs
      run: |
        TAR_FILENAME="hydra_outputs_${GITHUB_SHA_SHORT}.tar.gz"
        tar -czf $TAR_FILENAME ./outputs/
        tar -tzf $TAR_FILENAME
        echo "HYDRA_OUTPUTS_TAR=$TAR_FILENAME" >> $GITHUB_ENV

    - name: Upload config artifact 
      uses: actions/upload-artifact@v3
      with:
        name: hydra-outputs
        path: ${{ env.HYDRA_OUTPUTS_TAR }}