From 37fe6281445b4bbe1e0330c2831ac192f496122e Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Wed, 21 Aug 2024 23:08:00 +0800 Subject: [PATCH 1/3] Initial --- .github/workflows/velox_backend.yml | 8 ++++++++ dev/ci-velox-buildshared-centos-8.sh | 10 ---------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/velox_backend.yml b/.github/workflows/velox_backend.yml index 34ceb6d460a2..3d2b72fc0a8d 100644 --- a/.github/workflows/velox_backend.yml +++ b/.github/workflows/velox_backend.yml @@ -1034,6 +1034,14 @@ jobs: ./cpp/build/velox/benchmarks/ /root/.m2/repository/org/apache/arrow/ key: cache-velox-build-centos-8-${{ hashFiles('./cache-key') }} + - name: Setup java and maven + run: | + sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true + sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true + yum install sudo patch java-1.8.0-openjdk-devel wget -y + ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz + tar -xvf apache-maven-3.8.8-bin.tar.gz && mv apache-maven-3.8.8 /usr/lib/maven + echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV - name: Build Gluten native libraries if: steps.cache.outputs.cache-hit != 'true' run: | diff --git a/dev/ci-velox-buildshared-centos-8.sh b/dev/ci-velox-buildshared-centos-8.sh index f337185de379..b6b0cda02d28 100755 --- a/dev/ci-velox-buildshared-centos-8.sh +++ b/dev/ci-velox-buildshared-centos-8.sh @@ -2,16 +2,6 @@ set -e -sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true -sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true - -yum install sudo patch java-1.8.0-openjdk-devel wget -y -# Required by building arrow java. -wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz -tar -xvf apache-maven-3.8.8-bin.tar.gz && mv apache-maven-3.8.8 /usr/lib/maven -export PATH="${PATH}:/usr/lib/maven/bin" -echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV - source /opt/rh/gcc-toolset-9/enable ./dev/builddeps-veloxbe.sh --run_setup_script=OFF --enable_ep_cache=OFF --build_tests=ON \ --build_examples=ON --build_benchmarks=ON --build_protobuf=ON From e44b45af43f53545d4e470cc3aa95845a142b21d Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Thu, 22 Aug 2024 11:08:31 +0800 Subject: [PATCH 2/3] Move to a helper script --- .github/workflows/util/setup_helper.sh | 30 +++++++++++ .github/workflows/velox_backend.yml | 65 +++++------------------ .github/workflows/velox_backend_cache.yml | 6 +++ 3 files changed, 49 insertions(+), 52 deletions(-) create mode 100644 .github/workflows/util/setup_helper.sh diff --git a/.github/workflows/util/setup_helper.sh b/.github/workflows/util/setup_helper.sh new file mode 100644 index 000000000000..6f0942dfbe7a --- /dev/null +++ b/.github/workflows/util/setup_helper.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function install_maven { + ( + cd /opt/ + wget -nv https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz + tar -xvf apache-maven-3.8.8-bin.tar.gz && mv apache-maven-3.8.8 /usr/lib/maven + ) + echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV +} + +for cmd in "$@" +do + echo "Running: $cmd" + "$cmd" +done diff --git a/.github/workflows/velox_backend.yml b/.github/workflows/velox_backend.yml index 3d2b72fc0a8d..fc375c666a4f 100644 --- a/.github/workflows/velox_backend.yml +++ b/.github/workflows/velox_backend.yml @@ -44,6 +44,7 @@ env: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true MVN_CMD: 'mvn -ntp' WGET_CMD: 'wget -nv' + SETUP: 'bash .github/workflows/util/setup_helper.sh' concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} @@ -206,12 +207,9 @@ jobs: else yum update -y && yum install -y java-1.8.0-openjdk-devel wget fi - ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz - tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven + $SETUP install_maven - name: Set environment variables run: | - echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV if [ "${{ matrix.java }}" = "java-17" ]; then echo "JAVA_HOME=/usr/lib/jvm/java-17-openjdk" >> $GITHUB_ENV elif [ "${{ matrix.java }}" = "java-11" ]; then @@ -436,13 +434,9 @@ jobs: - name: Setup java and maven run: | yum update -y && yum install -y java-1.8.0-openjdk-devel wget git - ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz - tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven + $SETUP install_maven - name: Build for Uniffle 0.9.0 run: | - export MAVEN_HOME=/usr/lib/maven && \ - export PATH=${PATH}:${MAVEN_HOME}/bin && \ export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \ cd /opt && \ git clone -b v0.9.0 https://github.com/apache/incubator-uniffle.git && \ @@ -461,14 +455,10 @@ jobs: bash ./bin/start-coordinator.sh && bash ./bin/start-shuffle-server.sh - name: Build for Spark ${{ matrix.spark }} run: | - export MAVEN_HOME=/usr/lib/maven && \ - export PATH=${PATH}:${MAVEN_HOME}/bin && \ cd $GITHUB_WORKSPACE/ && \ $MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox -Puniffle -DskipTests - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 with uniffle-0.9.0 run: | - export MAVEN_HOME=/usr/lib/maven && \ - export PATH=${PATH}:${MAVEN_HOME}/bin && \ export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \ cd $GITHUB_WORKSPACE/tools/gluten-it && \ $MVN_CMD clean install -Pspark-3.2 -Puniffle && \ @@ -554,10 +544,7 @@ jobs: - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y - ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz - tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven - echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV + $SETUP install_maven - name: Get Ccache uses: actions/cache/restore@v3 with: @@ -614,10 +601,7 @@ jobs: - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y - ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz - tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven - echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV + $SETUP install_maven - name: Get Ccache uses: actions/cache/restore@v3 with: @@ -661,10 +645,7 @@ jobs: - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y - ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz - tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven - echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV + $SETUP install_maven - name: Get Ccache uses: actions/cache/restore@v3 with: @@ -722,10 +703,7 @@ jobs: - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y - ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz - tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven - echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV + $SETUP install_maven - name: Get Ccache uses: actions/cache/restore@v3 with: @@ -770,10 +748,7 @@ jobs: - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y - ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz - tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven - echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV + $SETUP install_maven - name: Get Ccache uses: actions/cache/restore@v3 with: @@ -831,10 +806,7 @@ jobs: - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y - ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz - tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven - echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV + $SETUP install_maven - name: Get Ccache uses: actions/cache/restore@v3 with: @@ -879,10 +851,7 @@ jobs: - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y - ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz - tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven - echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV + $SETUP install_maven - name: Get Ccache uses: actions/cache/restore@v3 with: @@ -939,10 +908,7 @@ jobs: - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y - ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz - tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven - echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV + $SETUP install_maven - name: Get Ccache uses: actions/cache/restore@v3 with: @@ -993,10 +959,7 @@ jobs: - name: Setup build dependency run: | yum install sudo patch java-1.8.0-openjdk-devel wget -y - ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz - tar -xvf apache-maven-3.8.8-bin.tar.gz - mv apache-maven-3.8.8 /usr/lib/maven - echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV + $SETUP install_maven - name: Get Ccache uses: actions/cache/restore@v3 with: @@ -1039,9 +1002,7 @@ jobs: sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true yum install sudo patch java-1.8.0-openjdk-devel wget -y - ${WGET_CMD} https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz - tar -xvf apache-maven-3.8.8-bin.tar.gz && mv apache-maven-3.8.8 /usr/lib/maven - echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV + $SETUP install_maven - name: Build Gluten native libraries if: steps.cache.outputs.cache-hit != 'true' run: | diff --git a/.github/workflows/velox_backend_cache.yml b/.github/workflows/velox_backend_cache.yml index 0cb922f65f10..df1c68b27926 100644 --- a/.github/workflows/velox_backend_cache.yml +++ b/.github/workflows/velox_backend_cache.yml @@ -77,6 +77,12 @@ jobs: ./cpp/build/velox/benchmarks/ /root/.m2/repository/org/apache/arrow/ key: cache-velox-build-centos-8-${{ hashFiles('./cache-key') }} + - name: Setup java and maven + run: | + sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true + sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true + yum install sudo patch java-1.8.0-openjdk-devel wget -y + bash .github/workflows/util/setup_helper.sh install_maven - name: Build Gluten native libraries if: steps.check-cache.outputs.cache-hit != 'true' run: | From d4fa450a71c26fb202c02a72ef1a3682be8d30fe Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Thu, 22 Aug 2024 12:10:14 +0800 Subject: [PATCH 3/3] Fix comment --- .github/workflows/util/install_spark_resources.sh | 3 +++ .github/workflows/util/setup_helper.sh | 2 ++ .github/workflows/velox_backend_cache.yml | 8 +++----- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/util/install_spark_resources.sh b/.github/workflows/util/install_spark_resources.sh index 242952d60528..e1645b170dd5 100755 --- a/.github/workflows/util/install_spark_resources.sh +++ b/.github/workflows/util/install_spark_resources.sh @@ -16,6 +16,9 @@ # Download Spark resources, required by some Spark UTs. The resource path should be set # for spark.test.home in mvn test. + +set -e + INSTALL_DIR=$GITHUB_WORKSPACE case "$1" in 3.2) diff --git a/.github/workflows/util/setup_helper.sh b/.github/workflows/util/setup_helper.sh index 6f0942dfbe7a..8b41d8326416 100644 --- a/.github/workflows/util/setup_helper.sh +++ b/.github/workflows/util/setup_helper.sh @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -e + function install_maven { ( cd /opt/ diff --git a/.github/workflows/velox_backend_cache.yml b/.github/workflows/velox_backend_cache.yml index df1c68b27926..44ee969cba4f 100644 --- a/.github/workflows/velox_backend_cache.yml +++ b/.github/workflows/velox_backend_cache.yml @@ -77,16 +77,14 @@ jobs: ./cpp/build/velox/benchmarks/ /root/.m2/repository/org/apache/arrow/ key: cache-velox-build-centos-8-${{ hashFiles('./cache-key') }} - - name: Setup java and maven + - name: Build Gluten native libraries + if: steps.check-cache.outputs.cache-hit != 'true' run: | + df -a sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true yum install sudo patch java-1.8.0-openjdk-devel wget -y bash .github/workflows/util/setup_helper.sh install_maven - - name: Build Gluten native libraries - if: steps.check-cache.outputs.cache-hit != 'true' - run: | - df -a bash dev/ci-velox-buildshared-centos-8.sh - name: Cache if: steps.check-cache.outputs.cache-hit != 'true'