From 56f21ef89bd71c0600f7b0045e28843c42363aa3 Mon Sep 17 00:00:00 2001 From: Christian Zentgraf Date: Thu, 12 Dec 2024 01:19:00 -0500 Subject: [PATCH] [native] Test standard runner prestocpp build --- .github/workflows/format-check.yml | 24 ++ .github/workflows/header-check.yml | 24 ++ .../workflows/linux-build-and-unit-test.yml | 243 ++++++++++++++++++ .github/workflows/macos-build.yml | 42 +++ .github/workflows/prestocpp-linux-build.yml | 54 ++++ .../scripts/setup-centos.sh | 6 + 6 files changed, 393 insertions(+) create mode 100644 .github/workflows/format-check.yml create mode 100644 .github/workflows/header-check.yml create mode 100644 .github/workflows/linux-build-and-unit-test.yml create mode 100644 .github/workflows/macos-build.yml create mode 100644 .github/workflows/prestocpp-linux-build.yml diff --git a/.github/workflows/format-check.yml b/.github/workflows/format-check.yml new file mode 100644 index 0000000000000..243658f7acf3f --- /dev/null +++ b/.github/workflows/format-check.yml @@ -0,0 +1,24 @@ +name: format-check + +on: + workflow_dispatch: + pull_request: + paths: + - 'presto-native-execution/**' + +jobs: + format-check: + runs-on: ubuntu-latest + container: + image: public.ecr.aws/oss-presto/velox-dev:check + steps: + - uses: actions/checkout@v4.2.2 + - name: Fix git permissions + # Usually actions/checkout does this but as we run in a container + # it doesn't work + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Check formatting + run: | + git fetch origin master + cd presto-native-execution + make format-check diff --git a/.github/workflows/header-check.yml b/.github/workflows/header-check.yml new file mode 100644 index 0000000000000..c3f9f5fb3cab5 --- /dev/null +++ b/.github/workflows/header-check.yml @@ -0,0 +1,24 @@ +name: header-check + +on: + workflow_dispatch: + pull_request: + paths: + - 'presto-native-execution/**' + +jobs: + header-check: + runs-on: ubuntu-latest + container: + image: public.ecr.aws/oss-presto/velox-dev:check + steps: + - uses: actions/checkout@v4.2.2 + - name: Fix git permissions + # Usually actions/checkout does this but as we run in a container + # it doesn't work + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Check license headers + run: | + git fetch origin master + cd presto-native-execution + make header-check diff --git a/.github/workflows/linux-build-and-unit-test.yml b/.github/workflows/linux-build-and-unit-test.yml new file mode 100644 index 0000000000000..5a0c0ef6c8d81 --- /dev/null +++ b/.github/workflows/linux-build-and-unit-test.yml @@ -0,0 +1,243 @@ + +name: linux-build-and-unit-test + +on: + workflow_dispatch: + pull_request: + +jobs: + prestocpp-linux-build-and-unit-test: + runs-on: ubuntu-22.04 + container: + image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090 + env: + CCACHE_DIR: "${{ github.workspace }}/ccache" + MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError" + MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" + MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" + MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" + steps: + - uses: actions/checkout@v4 + - name: Fix git permissions + # Usually actions/checkout does this but as we run in a container + # it doesn't work + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Update velox + run: | + cd presto-native-execution + make velox-submodule + + - name: Install prestocpp adapter dependencies + run: | + mkdir -p ${HOME}/adapter-deps/install + source /opt/rh/gcc-toolset-12/enable + set -xu + cd presto-native-execution + DEPENDENCY_DIR=${HOME}/adapter-deps PROMPT_ALWAYS_RESPOND=n ./scripts/setup-adapters.sh + + - name: Install github CLI for the ccache stash + run: ./presto-native-execution/scripts/setup-centos.sh install_gh + + - name: Calculate merge-base date for ccache + run: | + git fetch origin master + git show -s --format=%cd --date="format:%Y%m%d" $(git merge-base origin/master HEAD) | tee merge-base-date + + - uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-linux-prestocpp-${{ runner.os }}-${{ hashFiles('merge-base-date') }} + + - name: Zero ccache statistics + run: ccache -sz + + - name: Build engine + run: | + source /opt/rh/gcc-toolset-12/enable + cd presto-native-execution + cmake \ + -B _build/release \ + -GNinja \ + -DTREAT_WARNINGS_AS_ERRORS=1 \ + -DENABLE_ALL_WARNINGS=1 \ + -DCMAKE_BUILD_TYPE=Release \ + -DPRESTO_ENABLE_PARQUET=ON \ + -DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \ + -DPRESTO_ENABLE_JWT=ON \ + -DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS \ + -DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER \ + -DCMAKE_PREFIX_PATH=/usr/local \ + -DThrift_ROOT=/usr/local \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DMAX_LINK_JOBS=2 + ninja -C _build/release -j 4 + + - name: Ccache after + run: ccache -s + + - uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-linux-prestocpp-${{ runner.os }}-${{ hashFiles('merge-base-date') }} + + - name: Run Unit Tests + run: | + # Ensure transitive dependency libboost-iostreams is found. + ldconfig /usr/local/lib + cd presto-native-execution/_build/release + ctest -j 4 -VV --output-on-failure --exclude-regex velox.* + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: presto-native-build + path: | + presto-native-execution/_build/release/presto_cpp/main/presto_server + presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main + + linux-presto-e2e-tests: + needs: prestocpp-linux-build-and-unit-test + runs-on: ubuntu-22.04 + container: + image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090 + env: + MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError" + MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" + MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" + MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" + steps: + - uses: actions/checkout@v4 + - name: Fix git permissions + # Usually actions/checkout does this but as we run in a container + # it doesn't work + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: presto-native-build + path: presto-native-execution/_build/release + + - name: Install OpenJDK8 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '8' + + - name: Cache local Maven repository + id: cache-maven + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven-2- + + - name: Populate maven cache + if: steps.cache-maven.outputs.cache-hit != 'true' + run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies + + - name: Maven install + env: + MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" + MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" + run: | + for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s) + + - name: Run presto-native e2e tests + run: | + export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server" + export TESTFILES=`find ./presto-native-execution/src/test -type f -name 'TestPrestoNative*.java'` + # Convert file paths to comma separated class names + export TESTCLASSES= + for test_file in $TESTFILES + do + tmp=${test_file##*/} + test_class=${tmp%%\.*} + export TESTCLASSES="${TESTCLASSES},$test_class" + done + export TESTCLASSES=${TESTCLASSES#,} + echo "TESTCLASSES = $TESTCLASSES" + # TODO: neeed to enable remote function tests with + # "-Ppresto-native-execution-remote-functions" once + # > https://github.com/facebookincubator/velox/discussions/6163 + # is fixed. + + mvn test \ + ${MAVEN_TEST} \ + -pl 'presto-native-execution' \ + -Dtest="${TESTCLASSES}" \ + -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \ + -DDATA_DIR=${RUNNER_TEMP} \ + -Duser.timezone=America/Bahia_Banderas \ + -T1C + + linux-spark-e2e-tests: + needs: prestocpp-linux-build-and-unit-test + runs-on: ubuntu-22.04 + container: + image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090 + env: + MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError" + MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" + MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" + MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" + steps: + - uses: actions/checkout@v4 + - name: Fix git permissions + # Usually actions/checkout does this but as we run in a container + # it doesn't work + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: presto-native-build + path: presto-native-execution/_build/release + + - name: Check Java + run: java --version + + - name: Cache local Maven repository + id: cache-maven + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven-2- + + - name: Populate maven cache + if: steps.cache-maven.outputs.cache-hit != 'true' + run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies + + - name: Maven install + env: + MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" + MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" + run: | + for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s) + + - name: Run spark e2e tests + run: | + export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server" + export TESTFILES=`find ./presto-native-execution/src/test -type f -name 'TestPrestoSpark*.java'` + # Convert file paths to comma separated class names + export TESTCLASSES= + for test_file in $TESTFILES + do + tmp=${test_file##*/} + test_class=${tmp%%\.*} + export TESTCLASSES="${TESTCLASSES},$test_class" + done + export TESTCLASSES=${TESTCLASSES#,} + echo "TESTCLASSES = $TESTCLASSES" + mvn test \ + ${MAVEN_TEST} \ + -pl 'presto-native-execution' \ + -Dtest="${TESTCLASSES}" \ + -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \ + -DDATA_DIR=${RUNNER_TEMP} \ + -Duser.timezone=America/Bahia_Banderas \ + -T1C diff --git a/.github/workflows/macos-build.yml b/.github/workflows/macos-build.yml new file mode 100644 index 0000000000000..453d20bfc1395 --- /dev/null +++ b/.github/workflows/macos-build.yml @@ -0,0 +1,42 @@ +name: macos-build + +on: + workflow_dispatch: + pull_request: + paths: + - 'presto-native-execution/**' + +jobs: + macos-build: + runs-on: macos-13 + steps: + - uses: actions/checkout@v4.2.2 + - name: Fix git permissions + # Usually actions/checkout does this but as we run in a container + # it doesn't work + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Update submodules + run: | + git config --global --add safe.directory $(pwd) + cd presto-native-execution + make submodules + - uses: maxim-lobanov/setup-xcode@v1.6.0 + with: + xcode-version: 15 + - name: "Setup MacOS" + run: | + set -xu + mkdir ~/deps ~/deps-src + git clone --depth 1 https://github.com/Homebrew/brew ~/deps + PATH=~/deps/bin:${PATH} DEPENDENCY_DIR=~/deps-src INSTALL_PREFIX=~/deps PROMPT_ALWAYS_RESPOND=n ./presto-native-execution/scripts/setup-macos.sh + # Calculate the prefix path before we delete brew's repos and taps. + rm -rf ~/deps/.git ~/deps/Library/Taps/ # Reduce cache size by 70%. + rm -rf ~/deps-src + - name: "Build presto_cpp on MacOS" + run: | + clang --version + export INSTALL_PREFIX=~/deps + export PATH=~/deps/bin:$(brew --prefix m4)/bin:$(brew --prefix bison)/bin:${PATH} + cd presto-native-execution + cmake -B _build/debug -GNinja -DTREAT_WARNINGS_AS_ERRORS=1 -DENABLE_ALL_WARNINGS=1 -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + ninja -C _build/debug diff --git a/.github/workflows/prestocpp-linux-build.yml b/.github/workflows/prestocpp-linux-build.yml new file mode 100644 index 0000000000000..210437113f708 --- /dev/null +++ b/.github/workflows/prestocpp-linux-build.yml @@ -0,0 +1,54 @@ +name: prestocpp-linux-build + +on: + workflow_dispatch: + pull_request: + paths: + - 'presto-native-execution/**' + +jobs: + linux-build-all: + runs-on: ubuntu-22.04 + container: + image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090 + steps: + - uses: actions/checkout@v4.2.2 + - name: Fix git permissions + # Usually actions/checkout does this but as we run in a container + # it doesn't work + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Update velox + run: | + cd presto-native-execution + make velox-submodule + - name: Install all adapter dependencies + run: | + mkdir -p ${GITHUB_WORKSPACE}/adapter-deps/install + source /opt/rh/gcc-toolset-12/enable + set -xu + cd presto-native-execution + DEPENDENCY_DIR=${GITHUB_WORKSPACE}/adapter-deps PROMPT_ALWAYS_RESPOND=n ./velox/scripts/setup-adapters.sh + DEPENDENCY_DIR=${GITHUB_WORKSPACE}/adapter-deps PROMPT_ALWAYS_RESPOND=n ./scripts/setup-adapters.sh + - name: Build engine + run: | + source /opt/rh/gcc-toolset-12/enable + cd presto-native-execution + cmake \ + -B _build/debug \ + -GNinja \ + -DAWSSDK_ROOT_DIR=${GITHUB_WORKSPACE}/adapter-deps/install \ + -DTREAT_WARNINGS_AS_ERRORS=1 \ + -DENABLE_ALL_WARNINGS=1 \ + -DCMAKE_BUILD_TYPE=Debug \ + -DPRESTO_ENABLE_PARQUET=ON \ + -DPRESTO_ENABLE_S3=ON \ + -DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \ + -DPRESTO_ENABLE_JWT=ON \ + -DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS \ + -DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER \ + -DPRESTO_ENABLE_TESTING=OFF \ + -DCMAKE_PREFIX_PATH=/usr/local \ + -DThrift_ROOT=/usr/local \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DMAX_LINK_JOBS=2 + ninja -C _build/debug -j 4 diff --git a/presto-native-execution/scripts/setup-centos.sh b/presto-native-execution/scripts/setup-centos.sh index 5d237460bb404..ffea3861bffc9 100755 --- a/presto-native-execution/scripts/setup-centos.sh +++ b/presto-native-execution/scripts/setup-centos.sh @@ -54,6 +54,12 @@ function install_proxygen { cmake_install -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=ON } +function install_gh { + curl -L https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_amd64.rpm > gh_2.63.2_linux_amd64.rpm + rpm -iv gh_2.63.2_linux_amd64.rpm + rm -f gh_2.63.2_linux_amd64.rpm +} + function install_presto_deps { run_and_time install_presto_deps_from_package_managers run_and_time install_gperf