Flexibility in the deps (#165)

* update python deps

* add main to RayDP CI branches

* test multiple pyspark versions

* less changes

.github/workflows/raydp.yml

@@ -19,16 +19,17 @@ name: RayDP CI
 
 on:
  push:
- branches: [ master ]
+ branches: [ main, master ]
  pull_request:
- branches: [ master ]
+ branches: [ main, master ]
 
 jobs:
  build-and-test:
 
  strategy:
  matrix:
  os: [ ubuntu-latest, macos-latest ]
+ pyspark-version: [3.0.0, 3.1.2]
  python-version: [3.6, 3.7, 3.8]
 
  runs-on: ${{ matrix.os }}
@@ -51,7 +52,6 @@ jobs:
  if: matrix.os == 'ubuntu-latest'
  run: |
  sudo apt-get install -y mpich
-
  - name: Cache pip - Ubuntu
  if: matrix.os == 'ubuntu-latest'
  uses: actions/cache@v2
@@ -87,6 +87,7 @@ jobs:
  env:
  GITHUB_CI: 1
  run: |
+ pip install pyspark==${{ matrix.pyspark-version }}
  ./build.sh
  pip install dist/raydp-*.whl
  - name: Lint

core/pom.xml

@@ -152,7 +152,7 @@
  <plugin>
  <groupId>net.alchim31.maven</groupId>
  <artifactId>scala-maven-plugin</artifactId>
- <version>3.2.2</version>
+ <version>3.3.3</version>
  <executions>
  <execution>
  <id>scala-compile-first</id>

python/setup.py

@@ -92,11 +92,11 @@ def run(self):
  install_requires = [
  "numpy",
  "typing",
- "pandas == 1.1.4",
+ "pandas >= 1.1.4",
  "psutil",
  "pyarrow >= 0.10",
- "ray == 1.4.0",
- "pyspark >= 3.0.0, < 3.1.0",
+ "ray >= 1.4.0, < 1.5.0",
+ "pyspark >= 3.0.0",
  "netifaces"
  ]