[ZEPPELIN-6150] zeppelin.sh shall respect HADOOP_HOME to find hadoop command
#4144
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: core | |
| on: | |
| push: | |
| branches-ignore: | |
| - 'dependabot/**' | |
| pull_request: | |
| branches: | |
| - master | |
| - 'branch-*' | |
| env: | |
| # Disable keepAlive and pool | |
| # https://github.com/actions/virtual-environments/issues/1499#issuecomment-689467080 | |
| MAVEN_OPTS: >- | |
| -Xms1024M -Xmx2048M -XX:MaxMetaspaceSize=1024m -XX:-UseGCOverheadLimit | |
| -Dhttp.keepAlive=false | |
| -Dmaven.wagon.http.pool=false | |
| -Dmaven.wagon.http.retryHandler.count=3 | |
| MAVEN_ARGS: >- | |
| -B --no-transfer-progress | |
| ZEPPELIN_HELIUM_REGISTRY: helium | |
| SPARK_PRINT_LAUNCH_COMMAND: "true" | |
| SPARK_LOCAL_IP: 127.0.0.1 | |
| ZEPPELIN_LOCAL_IP: 127.0.0.1 | |
| # Use the bash login, because we are using miniconda | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| permissions: | |
| contents: read # to fetch code (actions/checkout) | |
| jobs: | |
| # test on core-modules (zeppelin-interpreter,zeppelin-zengine,zeppelin-server), | |
| # some interpreters are included, because zeppelin-server test depends on them: spark, shell & markdown | |
| core-modules: | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| java: [ 11 ] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: ${{ matrix.java }} | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.m2/repository | |
| !~/.m2/repository/org/apache/zeppelin/ | |
| ~/.spark-dist | |
| ~/.cache | |
| ~/conda_pkgs_dir | |
| key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-zeppelin- | |
| - name: install application with some interpreter | |
| run: ./mvnw install -Pbuild-distr -DskipTests -pl zeppelin-server,zeppelin-web,spark-submit,spark/scala-2.12,spark/scala-2.13,markdown,angular,shell -am -Pweb-classic -Phelium-dev -Pexamples ${MAVEN_ARGS} | |
| - name: install and test plugins | |
| run: ./mvnw package -pl zeppelin-plugins -amd ${MAVEN_ARGS} | |
| - name: Setup conda environment with python 3.9 and R | |
| uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| activate-environment: python_3_with_R | |
| environment-file: testing/env_python_3.9_with_R.yml | |
| python-version: 3.9 | |
| channels: conda-forge,defaults | |
| channel-priority: true | |
| auto-activate-base: false | |
| use-mamba: true | |
| - name: Make IRkernel available to Jupyter | |
| run: | | |
| R -e "IRkernel::installspec()" | |
| conda list | |
| conda info | |
| - name: run tests # skip spark test because we would run them in other CI | |
| run: ./mvnw verify -Pusing-packaged-distr -pl zeppelin-server,zeppelin-web,spark-submit,spark/scala-2.12,spark/scala-2.13,markdown,angular,shell -am -Pweb-classic -Phelium-dev -Pexamples -Dtests.to.exclude=**/org/apache/zeppelin/spark/* -DfailIfNoTests=false | |
| # test interpreter modules except spark, flink, python, rlang, jupyter | |
| interpreter-test-non-core: | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| java: [ 11 ] | |
| env: | |
| INTERPRETERS: 'hbase,jdbc,file,flink-cmd,cassandra,elasticsearch,bigquery,alluxio,livy,groovy,java,neo4j,sparql,mongodb,influxdb,shell' | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: ${{ matrix.java }} | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.m2/repository | |
| !~/.m2/repository/org/apache/zeppelin/ | |
| ~/.spark-dist | |
| ~/.cache | |
| key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-zeppelin- | |
| - name: install environment | |
| run: ./mvnw install -DskipTests -am -pl ${INTERPRETERS} ${MAVEN_ARGS} | |
| - name: Setup conda environment with python 3.9 and R | |
| uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| activate-environment: python_3_with_R_and_tensorflow | |
| environment-file: testing/env_python_3_with_R_and_tensorflow.yml | |
| python-version: 3.9 | |
| channels: conda-forge,defaults | |
| channel-priority: true | |
| auto-activate-base: false | |
| use-mamba: true | |
| - name: verify interpreter | |
| run: ./mvnw verify -am -pl ${INTERPRETERS} ${MAVEN_ARGS} | |
| # test interpreter modules for jupyter, python, rlang | |
| interpreter-test-jupyter-python-rlang: | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python: [ 3.9 ] | |
| java: [ 11 ] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: ${{ matrix.java }} | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.m2/repository | |
| !~/.m2/repository/org/apache/zeppelin/ | |
| ~/.spark-dist | |
| ~/.cache | |
| key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-zeppelin- | |
| - name: Setup conda environment with python ${{ matrix.python }} and R | |
| uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| activate-environment: python_3_with_R | |
| environment-file: testing/env_python_${{ matrix.python }}_with_R.yml | |
| python-version: ${{ matrix.python }} | |
| channels: conda-forge,defaults | |
| channel-priority: true | |
| auto-activate-base: false | |
| use-mamba: true | |
| - name: Make IRkernel available to Jupyter | |
| run: | | |
| R -e "IRkernel::installspec()" | |
| - name: install environment | |
| run: | | |
| ./mvnw install -DskipTests -pl python,rlang,zeppelin-jupyter-interpreter -am ${MAVEN_ARGS} | |
| - name: run tests with ${{ matrix.python }} | |
| run: | | |
| ./mvnw test -pl python,rlang,zeppelin-jupyter-interpreter -DfailIfNoTests=false ${MAVEN_ARGS} | |
| # zeppelin integration test except Spark & Flink | |
| zeppelin-integration-test: | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| java: [ 11 ] | |
| steps: | |
| # user/password => root/root | |
| - name: Start mysql | |
| run: sudo systemctl start mysql.service | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: ${{ matrix.java }} | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.m2/repository | |
| !~/.m2/repository/org/apache/zeppelin/ | |
| ~/.spark-dist | |
| ~/.cache | |
| key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-zeppelin- | |
| - name: install environment | |
| run: | | |
| ./mvnw install -DskipTests -Pintegration -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/scala-2.12,spark/scala-2.13,markdown,flink-cmd,flink/flink-scala-2.12,jdbc,shell -am -Pweb-classic -Pflink-117 ${MAVEN_ARGS} | |
| ./mvnw package -pl zeppelin-plugins -amd -DskipTests ${MAVEN_ARGS} | |
| - name: Setup conda environment with python 3.9 and R | |
| uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| activate-environment: python_3_with_R | |
| environment-file: testing/env_python_3_with_R.yml | |
| python-version: 3.9 | |
| channels: conda-forge,defaults | |
| channel-priority: true | |
| auto-activate-base: false | |
| use-mamba: true | |
| - name: Make IRkernel available to Jupyter | |
| run: | | |
| R -e "IRkernel::installspec()" | |
| - name: run tests | |
| run: ./mvnw test -pl zeppelin-interpreter-integration -Pintegration -DfailIfNoTests=false -Dtest=ZeppelinClientIntegrationTest,ZeppelinClientWithAuthIntegrationTest,ZSessionIntegrationTest,ShellIntegrationTest,JdbcIntegrationTest | |
| - name: Print zeppelin logs | |
| if: always() | |
| run: if [ -d "logs" ]; then cat logs/*; fi | |
| flink-test-and-flink-integration-test: | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python: [ 3.9 ] | |
| flink: [116, 117] | |
| include: | |
| # Flink 1.15 supports Python 3.6, 3.7, and 3.8 | |
| # https://nightlies.apache.org/flink/flink-docs-release-1.15/docs/dev/python/installation/ | |
| - python: 3.8 | |
| flink: 115 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Set up JDK 8 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: 11 | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.m2/repository | |
| !~/.m2/repository/org/apache/zeppelin/ | |
| ~/.spark-dist | |
| ~/.cache | |
| key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-zeppelin- | |
| - name: install environment for flink | |
| run: | | |
| ./mvnw install -DskipTests -am -pl flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -Pintegration ${MAVEN_ARGS} | |
| ./mvnw clean package -pl zeppelin-plugins -amd -DskipTests ${MAVEN_ARGS} | |
| - name: Setup conda environment with python ${{ matrix.python }} and R | |
| uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| activate-environment: python_3_with_flink | |
| environment-file: testing/env_python_3_with_flink_${{ matrix.flink }}.yml | |
| python-version: ${{ matrix.python }} | |
| channels: conda-forge,defaults | |
| channel-priority: true | |
| auto-activate-base: false | |
| use-mamba: true | |
| - name: run tests for flink | |
| run: ./mvnw verify -pl flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -am -Pintegration -DfailIfNoTests=false -Dtest=org.apache.zeppelin.flink.*Test,FlinkIntegrationTest${{ matrix.flink }} ${MAVEN_ARGS} | |
| - name: Print zeppelin logs | |
| if: always() | |
| run: if [ -d "logs" ]; then cat logs/*; fi | |
| spark-integration-test: | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| java: [ 11 ] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: ${{ matrix.java }} | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.m2/repository | |
| !~/.m2/repository/org/apache/zeppelin/ | |
| ~/.spark-dist | |
| ~/.cache | |
| key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-zeppelin- | |
| - name: install environment | |
| run: | | |
| ./mvnw install -DskipTests -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/scala-2.12,spark/scala-2.13,markdown -am -Pweb-classic -Pintegration ${MAVEN_ARGS} | |
| ./mvnw clean package -pl zeppelin-plugins -amd -DskipTests ${MAVEN_ARGS} | |
| - name: Setup conda environment with python 3.9 and R | |
| uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| activate-environment: python_3_with_R | |
| environment-file: testing/env_python_3_with_R.yml | |
| python-version: 3.9 | |
| channels: conda-forge,defaults | |
| channel-priority: true | |
| auto-activate-base: false | |
| use-mamba: true | |
| - name: Make IRkernel available to Jupyter | |
| run: | | |
| R -e "IRkernel::installspec()" | |
| - name: run tests | |
| run: ./mvnw test -pl zeppelin-interpreter-integration -Pintegration -Dtest=SparkSubmitIntegrationTest,ZeppelinSparkClusterTest32,SparkIntegrationTest32,ZeppelinSparkClusterTest33,SparkIntegrationTest33 -DfailIfNoTests=false ${MAVEN_ARGS} | |
| # test on spark for each spark version & scala version | |
| spark-test: | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python: [ 3.9 ] | |
| java: [ 11 ] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: ${{ matrix.java }} | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.m2/repository | |
| !~/.m2/repository/org/apache/zeppelin/ | |
| ~/.spark-dist | |
| ~/.cache | |
| key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-zeppelin- | |
| - name: install environment | |
| run: ./mvnw install -DskipTests -pl spark-submit,spark/scala-2.12,spark/scala-2.13 -am ${MAVEN_ARGS} | |
| - name: Setup conda environment with python ${{ matrix.python }} and R | |
| uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| activate-environment: python_3_with_R | |
| environment-file: testing/env_python_${{ matrix.python }}_with_R.yml | |
| python-version: ${{ matrix.python }} | |
| channels: conda-forge,defaults | |
| channel-priority: true | |
| auto-activate-base: false | |
| use-mamba: true | |
| - name: Make IRkernel available to Jupyter | |
| run: | | |
| R -e "IRkernel::installspec()" | |
| - name: run spark-3.3 tests with scala-2.12 and python-${{ matrix.python }} | |
| run: | | |
| rm -rf spark/interpreter/metastore_db | |
| ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.3 -Pspark-scala-2.12 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS} | |
| - name: run spark-3.3 tests with scala-2.13 and python-${{ matrix.python }} | |
| run: | | |
| rm -rf spark/interpreter/metastore_db | |
| ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.3 -Pspark-scala-2.13 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS} | |
| - name: run spark-3.4 tests with scala-2.13 and python-${{ matrix.python }} | |
| run: | | |
| rm -rf spark/interpreter/metastore_db | |
| ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.4 -Pspark-scala-2.13 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS} | |
| - name: run spark-3.5 tests with scala-2.13 and python-${{ matrix.python }} | |
| run: | | |
| rm -rf spark/interpreter/metastore_db | |
| ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.5 -Pspark-scala-2.13 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS} | |
| # Livy 0.8.0 tested with Spark 3.2 | |
| # https://github.com/apache/incubator-livy/blob/v0.8.0-incubating/dev/docker/livy-dev-spark/Dockerfile#L20 | |
| livy-0-8-with-spark-3-2-under-python39: | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Set up JDK 11 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: 11 | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.m2/repository | |
| !~/.m2/repository/org/apache/zeppelin/ | |
| ~/.spark-dist | |
| ~/.cache | |
| key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-zeppelin- | |
| - name: install environment | |
| run: | | |
| ./mvnw install -DskipTests -pl livy -am ${MAVEN_ARGS} | |
| ./testing/downloadSpark.sh "3.2.4" "3.2" | |
| ./testing/downloadLivy.sh "0.8.0-incubating" "2.12" | |
| - name: Setup conda environment with python 3.9 and R | |
| uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| activate-environment: python_39_with_R | |
| environment-file: testing/env_python_3.9_with_R.yml | |
| python-version: 3.9 | |
| channels: conda-forge,defaults | |
| channel-priority: true | |
| auto-activate-base: false | |
| use-mamba: true | |
| - name: Make IRkernel available to Jupyter | |
| run: | | |
| R -e "IRkernel::installspec()" | |
| - name: run tests | |
| run: | | |
| export SPARK_HOME=$PWD/spark-3.2.4-bin-hadoop3.2 | |
| export LIVY_HOME=$PWD/apache-livy-0.8.0-incubating_2.12-bin | |
| ./mvnw verify -pl livy ${MAVEN_ARGS} | |
| default-build: | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| java: [ 11 ] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: ${{ matrix.java }} | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.m2/repository | |
| !~/.m2/repository/org/apache/zeppelin/ | |
| ~/.spark-dist | |
| ~/.cache | |
| key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-zeppelin- | |
| - name: build without any profiles | |
| run: ./mvnw clean verify -DskipTests ${MAVEN_ARGS} |