add workflow cli #139
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths-ignore: ["docs/**", "scripts/**", "**.md", "*"] | |
| pull_request: | |
| branches: | |
| - main | |
| types: [opened, synchronize, labeled, reopened, ready_for_review] | |
| paths-ignore: ["docs/**", "scripts/**", "**.md", "*"] | |
| workflow_dispatch: | |
| inputs: | |
| debug_enabled: | |
| description: "Run with tmate.io debugging enabled" | |
| required: true | |
| type: boolean | |
| default: false | |
| run_build_images: | |
| description: "Run build-images job" | |
| required: false | |
| type: boolean | |
| default: false | |
| run_execute_workflow: | |
| description: "Run execute-workflow job" | |
| required: false | |
| type: boolean | |
| default: false | |
| workflow_execution_mode: | |
| description: "Workflow execution mode" | |
| required: false | |
| type: string | |
| default: "prod" | |
| defaults: | |
| run: | |
| shell: bash | |
| permissions: | |
| actions: write | |
| contents: read | |
| # concurrency: | |
| # group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }} | |
| # cancel-in-progress: true | |
| jobs: | |
| set-variables: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| debug: ${{ steps.set-variables.outputs.debug }} | |
| skip_ci: ${{ steps.set-variables.outputs.skip_ci }} | |
| skip_tests: ${{ steps.set-variables.outputs.skip_tests }} | |
| mode: ${{ steps.set-variables.outputs.mode }} | |
| steps: | |
| - name: Set action variables | |
| id: set-variables | |
| run: | | |
| DEBUG="false" | |
| MODE="prod" | |
| SKIP_CI="false" | |
| SKIP_TESTS="false" | |
| if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then | |
| DEBUG="${{ inputs.debug_enabled }}" | |
| MODE="${{ inputs.workflow_execution_mode }}" | |
| fi | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| if ${{ contains(github.event.pull_request.labels.*.name, 'skip-ci') }}; then | |
| SKIP_CI="true" | |
| fi | |
| if ${{ contains(github.event.pull_request.labels.*.name, 'skip-tests') }}; then | |
| SKIP_TESTS="true" | |
| fi | |
| if ${{ contains(github.event.pull_request.labels.*.name, 'actions-debug') }}; then | |
| DEBUG="true" | |
| fi | |
| if ${{ contains(github.event.pull_request.labels.*.name, 'workflow-mode-dev') }}; then | |
| MODE="dev" | |
| fi | |
| fi | |
| echo "DEBUG=$DEBUG" | |
| echo "MODE=$MODE" | |
| echo "SKIP_CI=$SKIP_CI" | |
| echo "SKIP_TESTS=$SKIP_TESTS" | |
| echo "DEBUG=$DEBUG" >> $GITHUB_OUTPUT | |
| echo "MODE=$MODE" >> $GITHUB_OUTPUT | |
| echo "SKIP_CI=$SKIP_CI" >> $GITHUB_OUTPUT | |
| echo "SKIP_TESTS=$SKIP_TESTS" >> $GITHUB_OUTPUT | |
| config-workflows: | |
| needs: [set-variables] | |
| if: ${{ needs.set-variables.outputs.skip_ci != 'true' && (contains(github.event.pull_request.labels.*.name, 'execute-workflow') || (github.event_name == 'workflow_dispatch' && inputs.run_execute_workflow)) }} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| config-path: ${{ steps.config-output.outputs.path }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Create Flyte config from YAML template | |
| id: yq-process | |
| uses: mikefarah/yq@master | |
| with: | |
| cmd: "yq e \ | |
| '.admin.endpoint = strenv(FLYTE_CLUSTER_ENDPOINT) | \ | |
| .storage.stow.config.project_id = strenv(GCP_PROJECT_ID) | \ | |
| .storage.stow.config.scopes = strenv(GCP_STORAGE_SCOPES) | \ | |
| .storage.container = strenv(GCP_STORAGE_CONTAINER)' \ | |
| .flyte/config-template.yaml > .flyte/config.yaml" | |
| env: | |
| FLYTE_CLUSTER_ENDPOINT: ${{ secrets.FLYTE_CLUSTER_ENDPOINT }} | |
| GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} | |
| GCP_STORAGE_SCOPES: ${{ secrets.GCP_STORAGE_SCOPES }} | |
| GCP_STORAGE_CONTAINER: ${{ secrets.GCP_STORAGE_CONTAINER }} | |
| - name: Upload Flyte config as an artifact | |
| id: config-output | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: flyte-config | |
| path: ${{ secrets.FLYTECTL_CONFIG }} | |
| test: | |
| runs-on: ubuntu-latest | |
| needs: [set-variables, config-workflows] | |
| if: ${{ needs.set-variables.outputs.skip_ci != 'true' && needs.set-variables.outputs.skip_tests != 'true' }} | |
| concurrency: | |
| group: test-${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }}-${{ needs.set-variables.outputs.mode }} | |
| cancel-in-progress: true | |
| strategy: | |
| matrix: | |
| python_version: ['3.10'] | |
| steps: | |
| - name: Check Variables | |
| run: | | |
| echo "SKIP_CI=${{ needs.set-variables.outputs.skip_ci }}" | |
| echo "SKIP_TESTS=${{ needs.set-variables.outputs.skip_tests }}" | |
| echo "DEBUG=${{ needs.set-variables.outputs.debug }}" | |
| echo "MODE=${{ needs.set-variables.outputs.mode }}" | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup environment | |
| uses: ./.github/actions/setup_environment | |
| with: | |
| python_version: ${{ matrix.python_version }} | |
| debug_enabled: ${{ needs.set-variables.outputs.debug }} | |
| - name: Download Flyte config | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: flyte-config | |
| path: .flyte/ | |
| - name: "Setup tmate debug session" | |
| uses: mxschmitt/action-tmate@v3 | |
| if: ${{ inputs.debug_enabled }} | |
| - name: Lint and typecheck | |
| run: | | |
| make lint-check | |
| - name: Run tests | |
| run: | | |
| make test-cov-xml | |
| - name: Upload coverage | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| fail_ci_if_error: false | |
| verbose: true | |
| build-images: | |
| needs: [set-variables] | |
| if: ${{ needs.set-variables.outputs.skip_ci != 'true' && (contains(github.event.pull_request.labels.*.name, 'build-images') || contains(github.event.pull_request.labels.*.name, 'execute-workflow') || (github.event_name == 'workflow_dispatch' && inputs.run_build_images)) }} | |
| uses: ./.github/workflows/build-images.yaml | |
| concurrency: | |
| group: bi-${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }}-${{ needs.set-variables.outputs.mode }} | |
| cancel-in-progress: true | |
| secrets: | |
| GCP_ARTIFACT_REGISTRY_PATH: ${{ secrets.GCP_ARTIFACT_REGISTRY_PATH }} | |
| with: | |
| debug_enabled: ${{ needs.set-variables.outputs.debug }} | |
| mode: ${{ needs.set-variables.outputs.mode }} | |
| execute-workflow: | |
| needs: [config-workflows, build-images, set-variables] | |
| if: ${{ needs.set-variables.outputs.skip_ci != 'true' && (contains(github.event.pull_request.labels.*.name, 'execute-workflow') || (github.event_name == 'workflow_dispatch' && inputs.run_execute_workflow)) }} | |
| runs-on: ubuntu-latest | |
| concurrency: | |
| group: ef-${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }}-${{ needs.set-variables.outputs.mode }} | |
| cancel-in-progress: true | |
| strategy: | |
| matrix: | |
| python_version: ['3.10'] | |
| env: | |
| FLYTECTL_CONFIG: ${{ secrets.FLYTECTL_CONFIG }} | |
| FLYTE_OAUTH_CLIENT_SECRET: ${{ secrets.FLYTE_OAUTH_CLIENT_SECRET }} | |
| WORKFLOW_IMAGE: ${{ vars.WORKFLOW_IMAGE }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Inject slug/short variables | |
| uses: rlespinasse/github-slug-action@v4 | |
| - name: Setup environment | |
| uses: ./.github/actions/setup_environment | |
| with: | |
| python_version: ${{ matrix.python_version }} | |
| debug_enabled: ${{ needs.set-variables.outputs.debug }} | |
| - name: Download Flyte config | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: flyte-config | |
| path: .flyte/ | |
| - name: Setup tmate debug session | |
| if: ${{ inputs.debug_enabled == 'true' }} | |
| uses: mxschmitt/action-tmate@v3 | |
| - name: Execute workflow | |
| id: execute | |
| run: | | |
| make run_${{ needs.set-variables.outputs.mode }} | |
| - name: Create config tarball | |
| id: save-hydra-outputs | |
| run: | | |
| TAR_FILENAME="hydra_outputs_${GITHUB_SHA_SHORT}.tar.gz" | |
| tar -czf $TAR_FILENAME ./outputs/ | |
| tar -tzf $TAR_FILENAME | |
| echo "HYDRA_OUTPUTS_TAR=$TAR_FILENAME" >> $GITHUB_ENV | |
| - name: Upload config artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: hydra-outputs | |
| path: ${{ env.HYDRA_OUTPUTS_TAR }} |