Add lakeFS data lake recipe for GCP & k3d #4
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Linting & Integration Test | |
| on: | |
| workflow_dispatch: | |
| workflow_call: | |
| pull_request: | |
| types: [opened, synchronize, ready_for_review] | |
| paths-ignore: ["**.md", "!pyproject.toml"] | |
| push: | |
| branches: | |
| - main | |
| paths-ignore: ["**.md", "!pyproject.toml"] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| id-token: write | |
| contents: read | |
| jobs: | |
| ubuntu-setup-python-environment: | |
| if: github.event.pull_request.draft == false | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| python-version: ["3.8", "3.9", "3.10", "3.11"] | |
| fail-fast: false | |
| uses: ./.github/workflows/setup-python-environment.yml | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| os: ${{ matrix.os }} | |
| secrets: inherit | |
| windows-setup-python-environment: | |
| if: github.event.pull_request.draft == false | |
| strategy: | |
| matrix: | |
| os: [windows-latest] | |
| python-version: ["3.8", "3.9", "3.10", "3.11"] | |
| fail-fast: false | |
| uses: ./.github/workflows/setup-python-environment.yml | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| os: ${{ matrix.os }} | |
| secrets: inherit | |
| macos-setup-python-environment: | |
| if: github.event.pull_request.draft == false | |
| strategy: | |
| matrix: | |
| os: [macos-latest] | |
| python-version: ["3.8", "3.9", "3.10", "3.11"] | |
| fail-fast: false | |
| uses: ./.github/workflows/setup-python-environment.yml | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| os: ${{ matrix.os }} | |
| secrets: inherit | |
| ubuntu-lint-unit-test: | |
| needs: ubuntu-setup-python-environment | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| python-version: ["3.8", "3.9", "3.10", "3.11"] | |
| fail-fast: false | |
| uses: ./.github/workflows/lint-unit-test.yml | |
| with: | |
| os: ${{ matrix.os }} | |
| python-version: ${{ matrix.python-version }} | |
| secrets: inherit | |
| windows-lint-unit-test: | |
| needs: windows-setup-python-environment | |
| strategy: | |
| matrix: | |
| os: [windows-latest] | |
| python-version: ["3.8", "3.9", "3.10", "3.11"] | |
| fail-fast: false | |
| uses: ./.github/workflows/lint-unit-test.yml | |
| with: | |
| os: ${{ matrix.os }} | |
| python-version: ${{ matrix.python-version }} | |
| secrets: inherit | |
| macos-lint-unit-test: | |
| needs: macos-setup-python-environment | |
| strategy: | |
| matrix: | |
| os: [macos-latest] | |
| python-version: ["3.8", "3.9", "3.10", "3.11"] | |
| fail-fast: false | |
| uses: ./.github/workflows/lint-unit-test.yml | |
| with: | |
| os: ${{ matrix.os }} | |
| python-version: ${{ matrix.python-version }} | |
| secrets: inherit | |
| spell-check: | |
| name: spell-check | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Actions Repository | |
| uses: actions/checkout@v2 | |
| - name: Spelling checker | |
| uses: crate-ci/typos@master | |
| with: | |
| files: "." | |
| config: ./.typos.toml | |
| localstack-aws-integration-test: | |
| uses: ./.github/workflows/aws-integration-test.yml | |
| secrets: inherit | |
| aws_test: | |
| name: aws_test | |
| runs-on: ubuntu-latest | |
| if: github.event.pull_request.head.repo.fork == false | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY }} | |
| AWS_DEFAULT_REGION: eu-central-1 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v2 | |
| - name: Install Terraform | |
| uses: hashicorp/setup-terraform@v2 | |
| with: | |
| terraform_version: 1.3.4 | |
| - name: Loop through subdirectories | |
| run: | | |
| for dir in $(ls -d src/mlstacks/terraform/* | grep -Fv -e "modules" -e ".github" | grep "aws*"); do | |
| (cd $dir && terraform init -backend-config="path=$dir") | |
| (cd $dir && terraform fmt -check) | |
| (cd $dir && terraform validate) | |
| (cd $dir && terraform plan -input=false) | |
| done | |
| gcp_test: | |
| name: gcp_test | |
| runs-on: ubuntu-latest | |
| if: github.event.pull_request.head.repo.fork == false | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@master | |
| - id: "auth" | |
| uses: "google-github-actions/auth@v0" | |
| with: | |
| credentials_json: "${{ secrets.GCP_CREDENTIALS }}" | |
| - name: "Set up Cloud SDK" | |
| uses: "google-github-actions/setup-gcloud@v0" | |
| - name: Install Terraform | |
| uses: hashicorp/setup-terraform@v2 | |
| with: | |
| terraform_version: 1.3.4 | |
| - name: Loop through subdirectories | |
| run: | | |
| for dir in $(ls -d src/mlstacks/terraform/* | grep -Fv -e "modules" -e ".github" | grep "gcp*"); do | |
| (cd $dir && terraform init -backend-config="path=$dir") | |
| (cd $dir && terraform fmt -check) | |
| (cd $dir && terraform validate) | |
| (cd $dir && terraform plan -input=false -var='project_id=something' -var='bucket_name=something') | |
| done | |
| azure_test: | |
| name: azure-test | |
| runs-on: ubuntu-latest | |
| if: github.event.pull_request.head.repo.fork == false | |
| env: | |
| ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }} | |
| ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }} | |
| ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }} | |
| ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v2 | |
| - name: Install Terraform | |
| uses: hashicorp/setup-terraform@v2 | |
| with: | |
| terraform_version: 1.3.4 | |
| - name: Loop through subdirectories | |
| run: | | |
| for dir in $(ls -d src/mlstacks/terraform/* | grep -Fv -e "modules" -e ".github" | grep "azure*"); do | |
| (cd $dir && terraform init -backend-config="path=$dir") | |
| (cd $dir && terraform fmt -check) | |
| (cd $dir && terraform validate) | |
| (cd $dir && terraform plan -input=false) | |
| done | |
| k3d_test: | |
| name: k3d_test | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v2 | |
| - name: Install k3d | |
| run: | | |
| curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash | |
| - name: Install Terraform | |
| uses: hashicorp/setup-terraform@v2 | |
| with: | |
| terraform_version: 1.3.4 | |
| - name: Loop through subdirectories | |
| run: | | |
| for dir in $(ls -d src/mlstacks/terraform/* | grep -Fv -e "modules" -e ".github" | grep "k3d*"); do | |
| (cd $dir && terraform init -backend-config="path=$dir") | |
| (cd $dir && terraform fmt -check) | |
| (cd $dir && terraform validate) | |
| (cd $dir && terraform plan -input=false) | |
| done |