raydp.yml

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

name: RayDP CI

on:
  push:
    branches: [ main, master ]
  pull_request:
    branches: [ main, master ]
  workflow_dispatch:

jobs:
  build-and-test:

    strategy:
      matrix:
        os: [ ubuntu-latest ]
        python-version: [3.7, 3.8, 3.9]
        spark-version: [3.1.3, 3.2.4, 3.3.2, 3.4.0]

    runs-on: ${{ matrix.os }}

    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Set up JDK 1.8
        uses: actions/setup-java@v1
        with:
          java-version: 1.8
      - name: Install extra dependencies for macOS
        if: matrix.os == 'macos-latest'
        run: |
          brew install pkg-config
          brew install libuv libomp mpich
      - name: Install extra dependencies for Ubuntu
        if: matrix.os == 'ubuntu-latest'
        run: |
          sudo apt-get install -y mpich
      - name: Cache pip - Ubuntu
        if: matrix.os == 'ubuntu-latest'
        uses: actions/cache@v2
        with:
          path: ~/.cache/pip
          key: ${{ matrix.os }}-${{ matrix.python-version }}-pip
      - name: Cache pip - MacOS
        if: matrix.os == 'macos-latest'
        uses: actions/cache@v2
        with:
          path: ~/Library/Caches/pip
          key: ${{ matrix.os }}-${{ matrix.python-version }}-pip
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install wheel
          pip install "numpy<1.24"
          pip install "pydantic<2.0"
          SUBVERSION=$(python -c 'import sys; print(sys.version_info[1])')
          if [ "$(uname -s)" == "Linux" ]
          then
            pip install torch==1.8.1+cpu -f https://download.pytorch.org/whl/torch_stable.html
          else
            pip install torch
          fi
          pip install pyarrow==6.0.1 ray[default]==2.4.0 pytest koalas tensorflow tabulate grpcio-tools wget
          pip install "xgboost_ray[default]<=0.1.13"
          pip install torchmetrics
          HOROVOD_WITH_GLOO=1
          HOROVOD_WITH_PYTORCH=1
          pip install horovod[pytorch,ray]
      - name: Cache Maven
        uses: actions/cache@v2
        with:
          path: ~/.m2
          key: ${{ matrix.os }}-m2-${{ hashFiles('core/pom.xml') }}
      - name: Build and install
        env:
          GITHUB_CI: 1
        run: |
          pip install pyspark==${{ matrix.spark-version }}
          ./build.sh
          pip install dist/raydp-*.whl
      - name: Lint
        run: |
          pip install pylint==2.8.3
          pylint --rcfile=python/pylintrc python/raydp
          pylint --rcfile=python/pylintrc examples/*.py
      - name: Test with pytest
        run: |
          ray start --head --num-cpus 6
          pytest python/raydp/tests/ -v
          ray stop --force
      - name: Test Examples
        run: |
          ray start --head
          python examples/raydp-submit.py
          ray stop
          python examples/pytorch_nyctaxi.py
          python examples/tensorflow_nyctaxi.py
          python examples/horovod_nyctaxi.py
          python examples/xgboost_ray_nyctaxi.py
          # python examples/raytrain_nyctaxi.py
          python examples/data_process.py