Add example about data aware scheduling #136
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: execute whirl ci for examples | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| jobs: | |
| shellcheck: # Job that list subdirectories of ./examples | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: run shellcheck | |
| uses: sudo-bot/action-shellcheck@latest | |
| with: | |
| # https://github.com/koalaman/shellcheck#how-to-use | |
| cli-args: "-x whirl" | |
| directories: # Job that list subdirectories of ./examples | |
| runs-on: ubuntu-latest | |
| outputs: | |
| # generate output name dir by using inner step output | |
| dir: ${{ steps.setdirs.outputs.dir }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - id: setdirs # Give it an id to handle to get step outputs in the outputs key above | |
| # run: echo "::set-output name=dir::$(ls -d ./examples/* | jq -R -s -c 'split("\n")[:-1]')" | |
| run: echo "dir=$(ls -d ./examples/* | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT | |
| # Define step output named dir based on ls command transformed to JSON thanks to jq | |
| examples: # Job that list subdirectories of ./examples | |
| runs-on: ubuntu-latest | |
| outputs: | |
| # generate output name example by using inner step output | |
| example: ${{ steps.setexamples.outputs.example }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - id: setexamples # Give it an id to handle to get step outputs in the outputs key above | |
| run: echo "example=$(ls -d ./examples/* | sed -r 's/\.\/examples\/(.*)/\1/g' | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT | |
| # Define step output named example based on ls command transformed to JSON thanks to jq | |
| whirl-ci-default-envs: | |
| needs: [directories] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 4 | |
| matrix: | |
| example_dir: ${{ fromJson(needs.directories.outputs.dir) }} | |
| python_version: ["3.9", "3.12"] | |
| airflow_version: ["2.2.5", "2.9.1"] | |
| exclude: | |
| - airflow_version: "2.2.5" | |
| python_version: "3.12" | |
| # Needs more memory than available on the runner | |
| - example_dir: ./examples/dbt-spark-example | |
| - example_dir: ./examples/spark-delta-sharing | |
| - example_dir: ./examples/spark-s3-to-hive | |
| # No default dag available for CI | |
| - example_dir: ./examples/airflow-datasets | |
| # Run without parallelism separately | |
| - example_dir: ./examples/dbt-example | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python_version }} | |
| AIRFLOW_VERSION: ${{ matrix.airflow_version }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Run whirl CI ${{ matrix.example_dir }} | |
| working-directory: ${{ matrix.example_dir }} | |
| run: | | |
| echo Run Ci from example directory ${{ matrix.example_dir }} | |
| ../../whirl ci | |
| whirl-ci-default-envs-from-root-dir: | |
| needs: [examples] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 4 | |
| matrix: | |
| example: ${{ fromJson(needs.examples.outputs.example) }} | |
| python_version: ["3.9", "3.12"] | |
| airflow_version: ["2.2.5", "2.9.1"] | |
| exclude: | |
| - airflow_version: "2.2.5" | |
| python_version: "3.12" | |
| # Needs more memory than available on the runner | |
| - example: dbt-spark-example | |
| - example: spark-delta-sharing | |
| - example: spark-s3-to-hive | |
| # No default dag available for CI | |
| - example: airflow-datasets | |
| # Run without parallelism separately | |
| - example: dbt-example | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python_version }} | |
| AIRFLOW_VERSION: ${{ matrix.airflow_version }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Run whirl CI example ${{ matrix.example }} | |
| run: | | |
| echo Run Ci for example ${{ matrix.example }} | |
| ./whirl -x ${{ matrix.example }} ci | |
| whirl-ci-dbt-example: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 | |
| matrix: | |
| python_version: ["3.9", "3.12"] | |
| airflow_version: ["2.2.5", "2.9.1"] | |
| exclude: | |
| - airflow_version: "2.2.5" | |
| python_version: "3.12" | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python_version }} | |
| AIRFLOW_VERSION: ${{ matrix.airflow_version }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Run whirl CI dbt example | |
| working-directory: ./examples/dbt-example | |
| run: | | |
| echo Run Ci from dbt-example directory | |
| ../../whirl ci | |
| whirl-ci-extra-env-spark-s3-to-postgres: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Run whirl CI non-default env | |
| working-directory: ./examples/spark-s3-to-postgres/ | |
| run: | | |
| echo Run Ci for example spark-s3-to-postgres with non-default env | |
| ../../whirl ci -e postgres-s3-spark | |
| whirl-ci-extra-env-api-to-s3: | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 4 | |
| matrix: | |
| python_version: ["3.9", "3.12"] | |
| airflow_version: ["2.2.5", "2.9.1"] | |
| exclude: | |
| - airflow_version: "2.2.5" | |
| python_version: "3.12" | |
| runs-on: ubuntu-latest | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python_version }} | |
| AIRFLOW_VERSION: ${{ matrix.airflow_version }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Run whirl CI api-to-s3 on k8s executor | |
| working-directory: ./examples/api-to-s3/ | |
| run: | | |
| echo Run Ci for example api-to-s3 with k8s executor env | |
| ../../whirl ci -e api-python-s3-k8s | |
| whirl-ci-extra-env-ha-scheduler: | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 4 | |
| matrix: | |
| python_version: ["3.9", "3.12"] | |
| airflow_version: ["2.2.5", "2.9.1"] | |
| exclude: | |
| - airflow_version: "2.2.5" | |
| python_version: "3.12" | |
| runs-on: ubuntu-latest | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python_version }} | |
| AIRFLOW_VERSION: ${{ matrix.airflow_version }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Run whirl CI ha-scheduler env | |
| working-directory: ./examples/external-airflow-db/ | |
| run: | | |
| echo Run Ci for example external-airflow-db with ha-scheduler env | |
| ../../whirl ci -e ha-scheduler |