This repository has been archived by the owner on Jul 10, 2024. It is now read-only.
SUBMARINE-1408. Fix ubuntu low disk space issue #3252
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Licensed to the Apache Software Foundation (ASF) under one or more | |
| # contributor license agreements. See the NOTICE file distributed with | |
| # this work for additional information regarding copyright ownership. | |
| # The ASF licenses this file to You under the Apache License, Version 2.0 | |
| # (the "License"); you may not use this file except in compliance with | |
| # the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| name: python-sdk | |
| # Trigger the workflow on push or pull request | |
| on: [push, pull_request] | |
| env: | |
| KUBERNETES_VERSION: "v1.25.3" | |
| jobs: | |
| check-style: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install dependencies | |
| run: | | |
| pip install --upgrade pip setuptools wheel | |
| pip install -r ./dev-support/style-check/python/lint-requirements.txt | |
| pip install -r ./dev-support/style-check/python/mypy-requirements.txt | |
| - name: List installed packages | |
| run: pip list | |
| - name: Check python sdk code style | |
| run: ./dev-support/style-check/python/lint.sh | |
| unit: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| matrix: | |
| python-version: ['3.7', '3.8', '3.9', '3.10'] | |
| tf-version: ['1.15.0', '2.6.0', '2.7.0', '2.8.0', '2.9.0'] | |
| exclude: | |
| - python-version: '3.8' | |
| tf-version: '1.15.0' | |
| - python-version: '3.9' | |
| tf-version: '1.15.0' | |
| - python-version: '3.10' | |
| tf-version: '1.15.0' | |
| - python-version: '3.10' | |
| tf-version: '2.6.0' | |
| - python-version: '3.10' | |
| tf-version: '2.7.0' | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup python environment | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install requirements | |
| run: | | |
| pip install --upgrade pip setuptools wheel | |
| pip install -r ./submarine-sdk/pysubmarine/github-actions/test-requirements.txt | |
| - name: Install pysubmarine with tf1 and pytorch | |
| if: ${{ matrix.tf-version == '1.15.0' }} | |
| run: | | |
| pip install --no-cache-dir -e ./submarine-sdk/pysubmarine/.[tf,pytorch] | |
| - name: Install pysubmarine with tf2 and pytorch | |
| if: startsWith(matrix.tf-version, '2.') | |
| run: | | |
| sed -i "s/tensorflow>=2.6.0,<2.10.0/tensorflow~=${{ matrix.tf-version }}/" ./submarine-sdk/pysubmarine/setup.py | |
| sed -i "s/tensorflow-estimator>=2.6.0,<2.10.0/tensorflow-estimator~=${{ matrix.tf-version }}/" ./submarine-sdk/pysubmarine/setup.py | |
| pip install --no-cache-dir -e ./submarine-sdk/pysubmarine/.[tf2,pytorch] | |
| - name: List installed packages | |
| run: pip list && pipdeptree | |
| - name: Run unit test | |
| working-directory: ./submarine-sdk/pysubmarine | |
| run: pytest -m "not e2e" --cov-report xml | |
| - uses: codecov/codecov-action@v3 | |
| with: | |
| flags: python-unit | |
| files: ./submarine-sdk/pysubmarine/coverage.xml | |
| name: codecov-unit | |
| integration: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| strategy: | |
| matrix: | |
| python-version: ['3.7', '3.8', '3.9', '3.10'] | |
| tf-version: ['1.15.0', '2.6.0', '2.7.0', '2.8.0', '2.9.0'] | |
| exclude: | |
| - python-version: '3.8' | |
| tf-version: '1.15.0' | |
| - python-version: '3.9' | |
| tf-version: '1.15.0' | |
| - python-version: '3.10' | |
| tf-version: '1.15.0' | |
| - python-version: '3.10' | |
| tf-version: '2.6.0' | |
| - python-version: '3.10' | |
| tf-version: '2.7.0' | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 1 | |
| - name: Free Disk Space | |
| run: bash ./.github/scripts/free-disk-space.sh | |
| - name: Setup python environment | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install tf1 dependencies | |
| if: ${{ matrix.tf-version == '1.15.0' }} | |
| run: | | |
| pip install --upgrade pip setuptools wheel | |
| pip install --no-cache-dir -e ./submarine-sdk/pysubmarine/.[tf,pytorch] | |
| pip install -r ./submarine-sdk/pysubmarine/github-actions/test-requirements.txt | |
| - name: Install tf2 dependencies | |
| if: startsWith(matrix.tf-version, '2.') | |
| run: | | |
| pip install --upgrade pip setuptools wheel | |
| sed -i "s/tensorflow>=2.6.0,<2.10.0/tensorflow~=${{ matrix.tf-version }}/" ./submarine-sdk/pysubmarine/setup.py | |
| sed -i "s/tensorflow-estimator>=2.6.0,<2.10.0/tensorflow-estimator~=${{ matrix.tf-version }}/" ./submarine-sdk/pysubmarine/setup.py | |
| pip install --no-cache-dir -e ./submarine-sdk/pysubmarine/.[tf2,pytorch] | |
| pip install -r ./submarine-sdk/pysubmarine/github-actions/test-requirements.txt | |
| pip cache purge | |
| - name: List installed packages | |
| run: pip list | |
| - name: Set up JDK 11 | |
| uses: actions/setup-java@v1 | |
| with: | |
| java-version: "11" | |
| - name: Set up Maven 3.6.3 | |
| uses: stCarolas/setup-maven@v4 | |
| with: | |
| maven-version: 3.6.3 | |
| - name: Check version | |
| run: | | |
| mvn --version | |
| java -version | |
| - name: Create kind cluster | |
| run: kind create cluster --config ./.github/config/kind-config-kind.yaml --wait 3m --image kindest/node:${KUBERNETES_VERSION} | |
| - name: Download Istio 1.17.1 | |
| run: | | |
| curl -L https://istio.io/downloadIstio | ISTIO_VERSION=1.17.1 sh - | |
| cd istio-1.17.1 | |
| echo "$PWD/bin" >> $GITHUB_PATH | |
| - name: Install Istio | |
| run: istioctl install -y --set values.global.proxy.resources.requests.cpu=10m | |
| - name: Show K8s cluster information | |
| run: | | |
| kubectl cluster-info | |
| kubectl version | |
| kubectl get pods -n kube-system | |
| echo "current-context:" $(kubectl config current-context) | |
| echo "environment-kubeconfig:" ${KUBECONFIG} | |
| # Cache maven (for submarine server) | |
| - uses: actions/cache@v3 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-maven- | |
| - name: Build Image locally | |
| run: | |
| .github/scripts/build-image-locally-v3.sh | |
| mvn clean | |
| - name: Start submarine | |
| run: bash ./.github/scripts/start-submarine.sh | |
| - name: Set CLI config for e2e pytest | |
| run: | | |
| submarine config set connection.hostname localhost | |
| submarine config set connection.port 8080 | |
| submarine config list | |
| - name: Run integration test | |
| working-directory: ./submarine-sdk/pysubmarine | |
| run: pytest -m "e2e" --cov-report xml | |
| - uses: codecov/codecov-action@v3 | |
| with: | |
| flags: python-integration | |
| files: ./submarine-sdk/pysubmarine/coverage.xml | |
| name: codecov-integration | |
| - name: Failure status | |
| run: | | |
| kubectl describe nodes | |
| kubectl get pods | |
| kubectl -n default get events --sort-by='{.lastTimestamp}' | |
| if: ${{ failure() }} | |
| sonarcloud: | |
| if: github.repository == 'apache/submarine' && github.event_name == 'push' && github.ref == 'refs/heads/master' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: SonarCloud Scan | |
| uses: SonarSource/sonarcloud-github-action@master | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }} | |
| with: | |
| projectBaseDir: submarine-sdk/pysubmarine | |
| args: > | |
| -Dsonar.organization=apache | |
| -Dsonar.projectKey=apache_submarine_python_sdk | |
| -Dsonar.python.version=3.7,3.8,3.9,3.10 |