[native] Test standard runner prestocpp build

.github/workflows/format-check.yml

@@ -0,0 +1,24 @@
+name: format-check
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'presto-native-execution/**'
+
+jobs:
+  format-check:
+    runs-on: ubuntu-latest
+    container:
+      image: public.ecr.aws/oss-presto/velox-dev:check
+    steps:
+      - uses: actions/checkout@v4.2.2
+      - name: Fix git permissions
+        # Usually actions/checkout does this but as we run in a container
+        # it doesn't work
+        run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
+      - name: Check formatting
+        run: |
+          git fetch origin master
+          cd presto-native-execution
+          make format-check

.github/workflows/header-check.yml

@@ -0,0 +1,24 @@
+name: header-check
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'presto-native-execution/**'
+
+jobs:
+  header-check:
+    runs-on: ubuntu-latest
+    container:
+      image: public.ecr.aws/oss-presto/velox-dev:check
+    steps:
+      - uses: actions/checkout@v4.2.2
+      - name: Fix git permissions
+        # Usually actions/checkout does this but as we run in a container
+        # it doesn't work
+        run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
+      - name: Check license headers
+        run: |
+          git fetch origin master
+          cd presto-native-execution
+          make header-check

.github/workflows/linux-build-and-unit-test.yml

@@ -0,0 +1,243 @@
+
+name: linux-build-and-unit-test
+
+on:
+  workflow_dispatch:
+  pull_request:
+
+jobs:
+  prestocpp-linux-build-and-unit-test:
+    runs-on: ubuntu-22.04
+    container:
+      image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
+    env:
+      CCACHE_DIR: "${{ github.workspace }}/ccache"
+      MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError"
+      MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
+      MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
+      MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Fix git permissions
+        # Usually actions/checkout does this but as we run in a container
+        # it doesn't work
+        run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
+
+      - name: Update velox
+        run: |
+          cd presto-native-execution
+          make velox-submodule
+
+      - name: Install prestocpp adapter dependencies
+        run: |
+          mkdir -p ${HOME}/adapter-deps/install
+          source /opt/rh/gcc-toolset-12/enable
+          set -xu
+          cd presto-native-execution
+          DEPENDENCY_DIR=${HOME}/adapter-deps PROMPT_ALWAYS_RESPOND=n ./scripts/setup-adapters.sh
+
+      - name: Install github CLI for the ccache stash
+        run: ./presto-native-execution/scripts/setup-centos.sh install_gh
+
+      - name: Calculate merge-base date for ccache
+        run: |
+          git fetch origin master
+          git show -s --format=%cd --date="format:%Y%m%d" $(git merge-base origin/master HEAD) | tee merge-base-date
+
+      - uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3
+        with:
+          path: '${{ env.CCACHE_DIR }}'
+          key: ccache-linux-prestocpp-${{ runner.os }}-${{ hashFiles('merge-base-date') }}
+
+      - name: Zero ccache statistics
+        run: ccache -sz
+
+      - name: Build engine
+        run: |
+          source /opt/rh/gcc-toolset-12/enable
+          cd presto-native-execution
+          cmake \
+            -B _build/release \
+            -GNinja \
+            -DTREAT_WARNINGS_AS_ERRORS=1 \
+            -DENABLE_ALL_WARNINGS=1 \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DPRESTO_ENABLE_PARQUET=ON \
+            -DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \
+            -DPRESTO_ENABLE_JWT=ON \
+            -DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS \
+            -DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER \
+            -DCMAKE_PREFIX_PATH=/usr/local \
+            -DThrift_ROOT=/usr/local \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+            -DMAX_LINK_JOBS=2
+          ninja -C _build/release -j 4
+
+      - name: Ccache after
+        run: ccache -s
+
+      - uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3
+        with:
+          path: '${{ env.CCACHE_DIR }}'
+          key: ccache-linux-prestocpp-${{ runner.os }}-${{ hashFiles('merge-base-date') }}
+
+      - name: Run Unit Tests
+        run: |
+          # Ensure transitive dependency libboost-iostreams is found.
+          ldconfig /usr/local/lib
+          cd presto-native-execution/_build/release
+          ctest -j 4 -VV --output-on-failure --exclude-regex velox.*
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: presto-native-build
+          path: |
+            presto-native-execution/_build/release/presto_cpp/main/presto_server
+            presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main
+
+  linux-presto-e2e-tests:
+    needs: prestocpp-linux-build-and-unit-test
+    runs-on: ubuntu-22.04
+    container:
+      image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
+    env:
+      MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError"
+      MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
+      MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
+      MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Fix git permissions
+        # Usually actions/checkout does this but as we run in a container
+        # it doesn't work
+        run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: presto-native-build
+          path: presto-native-execution/_build/release
+
+      - name: Install OpenJDK8
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '8'
+
+      - name: Cache local Maven repository
+        id: cache-maven
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-2-
+
+      - name: Populate maven cache
+        if: steps.cache-maven.outputs.cache-hit != 'true'
+        run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies
+
+      - name: Maven install
+        env:
+          MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
+          MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
+        run: |
+          for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s)
+
+      - name: Run presto-native e2e tests
+        run: |
+          export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server"
+          export TESTFILES=`find ./presto-native-execution/src/test -type f -name 'TestPrestoNative*.java'`
+          # Convert file paths to comma separated class names
+          export TESTCLASSES=
+          for test_file in $TESTFILES
+          do
+            tmp=${test_file##*/}
+            test_class=${tmp%%\.*}
+            export TESTCLASSES="${TESTCLASSES},$test_class"
+          done
+          export TESTCLASSES=${TESTCLASSES#,}
+          echo "TESTCLASSES = $TESTCLASSES"
+          # TODO: neeed to enable remote function tests with
+          # "-Ppresto-native-execution-remote-functions" once
+          # > https://github.com/facebookincubator/velox/discussions/6163
+          # is fixed.
+
+          mvn test \
+            ${MAVEN_TEST} \
+            -pl 'presto-native-execution' \
+            -Dtest="${TESTCLASSES}" \
+            -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \
+            -DDATA_DIR=${RUNNER_TEMP} \
+            -Duser.timezone=America/Bahia_Banderas \
+            -T1C
+
+  linux-spark-e2e-tests:
+    needs: prestocpp-linux-build-and-unit-test
+    runs-on: ubuntu-22.04
+    container:
+      image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
+    env:
+      MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError"
+      MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
+      MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
+      MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Fix git permissions
+        # Usually actions/checkout does this but as we run in a container
+        # it doesn't work
+        run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: presto-native-build
+          path: presto-native-execution/_build/release
+
+      - name: Check Java
+        run: java --version
+
+      - name: Cache local Maven repository
+        id: cache-maven
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-2-
+
+      - name: Populate maven cache
+        if: steps.cache-maven.outputs.cache-hit != 'true'
+        run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies
+
+      - name: Maven install
+        env:
+          MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
+          MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
+        run: |
+          for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s)
+
+      - name: Run spark e2e tests
+        run: |
+          export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server"
+          export TESTFILES=`find ./presto-native-execution/src/test -type f -name 'TestPrestoSpark*.java'`
+          # Convert file paths to comma separated class names
+          export TESTCLASSES=
+          for test_file in $TESTFILES
+          do
+            tmp=${test_file##*/}
+            test_class=${tmp%%\.*}
+            export TESTCLASSES="${TESTCLASSES},$test_class"
+          done
+          export TESTCLASSES=${TESTCLASSES#,}
+          echo "TESTCLASSES = $TESTCLASSES"
+          mvn test \
+            ${MAVEN_TEST} \
+            -pl 'presto-native-execution' \
+            -Dtest="${TESTCLASSES}" \
+            -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \
+            -DDATA_DIR=${RUNNER_TEMP} \
+            -Duser.timezone=America/Bahia_Banderas \
+            -T1C

.github/workflows/macos-build.yml

@@ -0,0 +1,42 @@
+name: macos-build
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'presto-native-execution/**'
+
+jobs:
+  macos-build:
+    runs-on: macos-13
+    steps:
+      - uses: actions/checkout@v4.2.2
+      - name: Fix git permissions
+        # Usually actions/checkout does this but as we run in a container
+        # it doesn't work
+        run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
+      - name: Update submodules
+        run: |
+          git config --global --add safe.directory $(pwd)
+          cd presto-native-execution
+          make submodules
+      - uses: maxim-lobanov/setup-xcode@v1.6.0
+        with:
+          xcode-version: 15
+      - name: "Setup MacOS"
+        run: |
+          set -xu
+          mkdir ~/deps ~/deps-src
+          git clone --depth 1 https://github.com/Homebrew/brew ~/deps
+          PATH=~/deps/bin:${PATH} DEPENDENCY_DIR=~/deps-src INSTALL_PREFIX=~/deps PROMPT_ALWAYS_RESPOND=n ./presto-native-execution/scripts/setup-macos.sh
+          # Calculate the prefix path before we delete brew's repos and taps.
+          rm -rf ~/deps/.git ~/deps/Library/Taps/  # Reduce cache size by 70%.
+          rm -rf ~/deps-src
+      - name: "Build presto_cpp on MacOS"
+        run: |
+          clang --version
+          export INSTALL_PREFIX=~/deps
+          export PATH=~/deps/bin:$(brew --prefix m4)/bin:$(brew --prefix bison)/bin:${PATH}
+          cd presto-native-execution
+          cmake -B _build/debug -GNinja -DTREAT_WARNINGS_AS_ERRORS=1 -DENABLE_ALL_WARNINGS=1 -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+          ninja -C _build/debug

.github/workflows/prestocpp-linux-build.yml

@@ -0,0 +1,54 @@
+name: prestocpp-linux-build
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'presto-native-execution/**'
+
+jobs:
+  linux-build-all:
+    runs-on: ubuntu-22.04
+    container:
+      image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
+    steps:
+      - uses: actions/checkout@v4.2.2
+      - name: Fix git permissions
+        # Usually actions/checkout does this but as we run in a container
+        # it doesn't work
+        run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
+      - name: Update velox
+        run: |
+          cd presto-native-execution
+          make velox-submodule
+      - name: Install all adapter dependencies
+        run: |
+          mkdir -p ${GITHUB_WORKSPACE}/adapter-deps/install
+          source /opt/rh/gcc-toolset-12/enable
+          set -xu
+          cd presto-native-execution
+          DEPENDENCY_DIR=${GITHUB_WORKSPACE}/adapter-deps PROMPT_ALWAYS_RESPOND=n ./velox/scripts/setup-adapters.sh
+          DEPENDENCY_DIR=${GITHUB_WORKSPACE}/adapter-deps PROMPT_ALWAYS_RESPOND=n ./scripts/setup-adapters.sh
+      - name: Build engine
+        run: |
+          source /opt/rh/gcc-toolset-12/enable
+          cd presto-native-execution
+          cmake \
+            -B _build/debug \
+            -GNinja \
+            -DAWSSDK_ROOT_DIR=${GITHUB_WORKSPACE}/adapter-deps/install \
+            -DTREAT_WARNINGS_AS_ERRORS=1 \
+            -DENABLE_ALL_WARNINGS=1 \
+            -DCMAKE_BUILD_TYPE=Debug \
+            -DPRESTO_ENABLE_PARQUET=ON \
+            -DPRESTO_ENABLE_S3=ON \
+            -DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \
+            -DPRESTO_ENABLE_JWT=ON \
+            -DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS \
+            -DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER \
+            -DPRESTO_ENABLE_TESTING=OFF \
+            -DCMAKE_PREFIX_PATH=/usr/local \
+            -DThrift_ROOT=/usr/local \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+            -DMAX_LINK_JOBS=2
+          ninja -C _build/debug -j 4