[WIP] Add native row expression optimizer #162
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: prestocpp-linux-build-and-unit-test | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| paths: | |
| - 'presto-native-execution/**' | |
| - '.github/workflows/prestocpp-linux-build-and-unit-test.yml' | |
| jobs: | |
| prestocpp-linux-build-for-test: | |
| runs-on: ubuntu-22.04 | |
| container: | |
| image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090 | |
| env: | |
| CCACHE_DIR: "${{ github.workspace }}/ccache" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Fix git permissions | |
| # Usually actions/checkout does this but as we run in a container | |
| # it doesn't work | |
| run: git config --global --add safe.directory ${GITHUB_WORKSPACE} | |
| - name: Update velox | |
| run: | | |
| cd presto-native-execution | |
| make velox-submodule | |
| - name: Install Github CLI for using apache/infrastructure-actions/stash | |
| run: | | |
| curl -L https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_amd64.rpm > gh_2.63.2_linux_amd64.rpm | |
| rpm -iv gh_2.63.2_linux_amd64.rpm | |
| - uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3 | |
| with: | |
| path: '${{ env.CCACHE_DIR }}' | |
| key: ccache-prestocpp-linux-build-for-test | |
| - name: Zero ccache statistics | |
| run: ccache -sz | |
| - name: Build engine | |
| run: | | |
| source /opt/rh/gcc-toolset-12/enable | |
| cd presto-native-execution | |
| cmake \ | |
| -B _build/release \ | |
| -GNinja \ | |
| -DTREAT_WARNINGS_AS_ERRORS=1 \ | |
| -DENABLE_ALL_WARNINGS=1 \ | |
| -DCMAKE_BUILD_TYPE=Release \ | |
| -DPRESTO_ENABLE_PARQUET=ON \ | |
| -DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \ | |
| -DPRESTO_ENABLE_JWT=ON \ | |
| -DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS \ | |
| -DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER \ | |
| -DCMAKE_PREFIX_PATH=/usr/local \ | |
| -DThrift_ROOT=/usr/local \ | |
| -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ | |
| -DMAX_LINK_JOBS=4 | |
| ninja -C _build/release -j 4 | |
| - name: Ccache after | |
| run: ccache -s | |
| - uses: apache/infrastructure-actions/stash/save@4ab8682fbd4623d2b4fc1c98db38aba5091924c3 | |
| with: | |
| path: '${{ env.CCACHE_DIR }}' | |
| key: ccache-prestocpp-linux-build-for-test | |
| - name: Run Unit Tests | |
| run: | | |
| # Ensure transitive dependency libboost-iostreams is found. | |
| ldconfig /usr/local/lib | |
| cd presto-native-execution/_build/release | |
| ctest -j 4 -VV --output-on-failure --exclude-regex velox.* | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: presto-native-build | |
| path: | | |
| presto-native-execution/_build/release/presto_cpp/main/presto_server | |
| presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main | |
| prestocpp-linux-presto-e2e-tests: | |
| needs: prestocpp-linux-build-for-test | |
| runs-on: ubuntu-22.04 | |
| container: | |
| image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090 | |
| env: | |
| MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError" | |
| MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" | |
| MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Fix git permissions | |
| # Usually actions/checkout does this but as we run in a container | |
| # it doesn't work | |
| run: git config --global --add safe.directory ${GITHUB_WORKSPACE} | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: presto-native-build | |
| path: presto-native-execution/_build/release | |
| # Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38 | |
| - name: Restore execute permissions and library path | |
| run: | | |
| chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server | |
| chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main | |
| # Ensure transitive dependency libboost-iostreams is found. | |
| ldconfig /usr/local/lib | |
| - name: Install OpenJDK8 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '8' | |
| - name: Cache local Maven repository | |
| id: cache-maven | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-maven-2- | |
| - name: Populate maven cache | |
| if: steps.cache-maven.outputs.cache-hit != 'true' | |
| run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies | |
| - name: Maven install | |
| env: | |
| # Use different Maven options to install. | |
| MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" | |
| run: | | |
| for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s) | |
| - name: Run presto-native e2e tests | |
| run: | | |
| export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server" | |
| export TESTFILES=`find ./presto-native-execution/src/test -type f -name 'TestPrestoNative*.java'` | |
| # Convert file paths to comma separated class names | |
| export TESTCLASSES= | |
| for test_file in $TESTFILES | |
| do | |
| tmp=${test_file##*/} | |
| test_class=${tmp%%\.*} | |
| export TESTCLASSES="${TESTCLASSES},$test_class" | |
| done | |
| export TESTCLASSES=${TESTCLASSES#,} | |
| echo "TESTCLASSES = $TESTCLASSES" | |
| # TODO: neeed to enable remote function tests with | |
| # "-Ppresto-native-execution-remote-functions" once | |
| # > https://github.com/facebookincubator/velox/discussions/6163 | |
| # is fixed. | |
| mvn test \ | |
| ${MAVEN_TEST} \ | |
| -pl 'presto-native-execution' \ | |
| -Dtest="${TESTCLASSES}" \ | |
| -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \ | |
| -DDATA_DIR=${RUNNER_TEMP} \ | |
| -Duser.timezone=America/Bahia_Banderas \ | |
| -T1C | |
| prestocpp-linux-spark-e2e-tests: | |
| needs: prestocpp-linux-build-for-test | |
| runs-on: ubuntu-22.04 | |
| container: | |
| image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090 | |
| env: | |
| MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError" | |
| MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" | |
| MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Fix git permissions | |
| # Usually actions/checkout does this but as we run in a container | |
| # it doesn't work | |
| run: git config --global --add safe.directory ${GITHUB_WORKSPACE} | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: presto-native-build | |
| path: presto-native-execution/_build/release | |
| # Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38 | |
| - name: Restore execute permissions and library path | |
| run: | | |
| chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server | |
| chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main | |
| # Ensure transitive dependency libboost-iostreams is found. | |
| ldconfig /usr/local/lib | |
| - name: Check Java | |
| run: java --version | |
| - name: Cache local Maven repository | |
| id: cache-maven | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-maven-2- | |
| - name: Populate maven cache | |
| if: steps.cache-maven.outputs.cache-hit != 'true' | |
| run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies | |
| - name: Maven install | |
| env: | |
| # Use different Maven options to install. | |
| MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" | |
| run: | | |
| for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s) | |
| - name: Run spark e2e tests | |
| run: | | |
| export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server" | |
| export TESTFILES=`find ./presto-native-execution/src/test -type f -name 'TestPrestoSpark*.java'` | |
| # Convert file paths to comma separated class names | |
| export TESTCLASSES= | |
| for test_file in $TESTFILES | |
| do | |
| tmp=${test_file##*/} | |
| test_class=${tmp%%\.*} | |
| export TESTCLASSES="${TESTCLASSES},$test_class" | |
| done | |
| export TESTCLASSES=${TESTCLASSES#,} | |
| echo "TESTCLASSES = $TESTCLASSES" | |
| mvn test \ | |
| ${MAVEN_TEST} \ | |
| -pl 'presto-native-execution' \ | |
| -Dtest="${TESTCLASSES}" \ | |
| -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \ | |
| -DDATA_DIR=${RUNNER_TEMP} \ | |
| -Duser.timezone=America/Bahia_Banderas \ | |
| -T1C | |
| prestocpp-linux-presto-sidecar-tests: | |
| needs: prestocpp-linux-build-for-test | |
| runs-on: ubuntu-22.04 | |
| container: | |
| image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090 | |
| env: | |
| MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError" | |
| MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" | |
| MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Fix git permissions | |
| # Usually actions/checkout does this but as we run in a container | |
| # it doesn't work | |
| run: git config --global --add safe.directory ${GITHUB_WORKSPACE} | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: presto-native-build | |
| path: presto-native-execution/_build/release | |
| # Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38 | |
| - name: Restore execute permissions and library path | |
| run: | | |
| chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server | |
| chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main | |
| # Ensure transitive dependency libboost-iostreams is found. | |
| ldconfig /usr/local/lib | |
| - name: Install OpenJDK8 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '8' | |
| - name: Cache local Maven repository | |
| id: cache-maven | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-maven-2- | |
| - name: Populate maven cache | |
| if: steps.cache-maven.outputs.cache-hit != 'true' | |
| run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies | |
| - name: Maven install | |
| env: | |
| # Use different Maven options to install. | |
| MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" | |
| run: | | |
| for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s) | |
| - name: Run presto-native sidecar tests | |
| run: | | |
| export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server" | |
| export TESTFILES=`find ./presto-native-sidecar-plugin/src/test -type f -name 'Test*.java'` | |
| # Convert file paths to comma separated class names | |
| export TESTCLASSES= | |
| for test_file in $TESTFILES | |
| do | |
| tmp=${test_file##*/} | |
| test_class=${tmp%%\.*} | |
| export TESTCLASSES="${TESTCLASSES},$test_class" | |
| done | |
| export TESTCLASSES=${TESTCLASSES#,} | |
| echo "TESTCLASSES = $TESTCLASSES" | |
| mvn test \ | |
| ${MAVEN_TEST} \ | |
| -pl 'presto-native-sidecar-plugin' \ | |
| -Dtest="${TESTCLASSES}" \ | |
| -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \ | |
| -DDATA_DIR=${RUNNER_TEMP} \ | |
| -Duser.timezone=America/Bahia_Banderas \ | |
| -T1C |