adding wheel build for libcudf

.github/workflows/build.yaml

@@ -67,7 +67,27 @@ jobs:
  node_type: "gpu-v100-latest-1"
  run_script: "ci/build_docs.sh"
  sha: ${{ inputs.sha }}
+ wheel-build-libcudf:
+ secrets: inherit
+ uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10
+ with:
+ build_type: ${{ inputs.build_type || 'branch' }}
+ branch: ${{ inputs.branch }}
+ sha: ${{ inputs.sha }}
+ date: ${{ inputs.date }}
+ script: ci/build_wheel_libcudf.sh
+ wheel-publish-libcudf:
+ needs: wheel-build-libcudf
+ secrets: inherit
+ uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10
+ with:
+ build_type: ${{ inputs.build_type || 'branch' }}
+ branch: ${{ inputs.branch }}
+ sha: ${{ inputs.sha }}
+ date: ${{ inputs.date }}
+ package-name: libcudf
  wheel-build-pylibcudf:
+ needs: [wheel-publish-libcudf]
  secrets: inherit
  uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10
  with:
@@ -87,7 +107,7 @@ jobs:
  date: ${{ inputs.date }}
  package-name: pylibcudf
  wheel-build-cudf:
- needs: wheel-publish-pylibcudf
+ needs: [wheel-publish-libcudf, wheel-publish-pylibcudf]
  secrets: inherit
  uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10
  with:

.github/workflows/pr.yaml

@@ -23,6 +23,7 @@ jobs:
  - static-configure
  - conda-notebook-tests
  - docs-build
+ - wheel-build-libcudf
  - wheel-build-pylibcudf
  - wheel-build-cudf
  - wheel-tests-cudf
@@ -121,15 +122,23 @@ jobs:
  arch: "amd64"
  container_image: "rapidsai/ci-conda:latest"
  run_script: "ci/build_docs.sh"
- wheel-build-pylibcudf:
+ wheel-build-libcudf:
  needs: checks
  secrets: inherit
  uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10
+ with:
+ matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
+ build_type: pull-request
+ script: "ci/build_wheel_libcudf.sh"
+ wheel-build-pylibcudf:
+ needs: [checks, wheel-build-libcudf]
+ secrets: inherit
+ uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10
  with:
  build_type: pull-request
  script: "ci/build_wheel_pylibcudf.sh"
  wheel-build-cudf:
- needs: wheel-build-pylibcudf
+ needs: [checks, wheel-build-libcudf, wheel-build-pylibcudf]
  secrets: inherit
  uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10
  with:
@@ -143,7 +152,7 @@ jobs:
  build_type: pull-request
  script: ci/test_wheel_cudf.sh
  wheel-build-cudf-polars:
- needs: wheel-build-pylibcudf
+ needs: [wheel-build-pylibcudf, wheel-build-cudf]
  secrets: inherit
  uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10
  with:

build.sh

@@ -17,13 +17,14 @@ ARGS=$*
 # script, and that this script resides in the repo dir!
 REPODIR=$(cd $(dirname $0); pwd)
 
-VALIDARGS="clean libcudf pylibcudf cudf cudfjar dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz -v -g -n --pydevelop -l --allgpuarch --disable_nvtx --opensource_nvcomp --show_depr_warn --ptds -h --build_metrics --incl_cache_stats --disable_large_strings"
-HELP="$0 [clean] [libcudf] [pylibcudf] [cudf] [cudfjar] [dask_cudf] [benchmarks] [tests] [libcudf_kafka] [cudf_kafka] [custreamz] [-v] [-g] [-n] [-h] [--cmake-args=\\\"<args>\\\"]
+VALIDARGS="clean libcudf libcudfwheel pylibcudf cudf cudfjar dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz -v -g -n --pydevelop -l --allgpuarch --disable_nvtx --opensource_nvcomp --show_depr_warn --ptds -h --build_metrics --incl_cache_stats --disable_large_strings"
+HELP="$0 [clean] [libcudf] [libcudfwheel] [pylibcudf] [cudf] [cudfjar] [dask_cudf] [benchmarks] [tests] [libcudf_kafka] [cudf_kafka] [custreamz] [-v] [-g] [-n] [-h] [--cmake-args=\\\"<args>\\\"]
  clean - remove all existing build artifacts and configuration (start
  over)
  libcudf - build the cudf C++ code only
  pylibcudf - build the pylibcudf Python package
  cudf - build the cudf Python package
+ libcudfwheel - build the cudf C++ code packaged as a python wheel package
  cudfjar - build cudf JAR with static libcudf using devtoolset toolchain
  dask_cudf - build the dask_cudf Python package
  benchmarks - build benchmarks
@@ -341,6 +342,14 @@ if buildAll || hasArg libcudf; then
  fi
 fi
 
+# Build and install the libcudf Python package
+if buildAll || hasArg libcudfwheel; then
+
+ cd ${REPODIR}/python/libcudf
+ SKBUILD_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${INSTALL_PREFIX};-DCMAKE_LIBRARY_PATH=${LIBCUDF_BUILD_DIR};-DCMAKE_CUDA_ARCHITECTURES=${CUDF_CMAKE_CUDA_ARCHITECTURES};${EXTRA_CMAKE_ARGS}" \
+ python ${PYTHON_ARGS_FOR_INSTALL} .
+fi
+
 # Build and install the pylibcudf Python package
 if buildAll || hasArg pylibcudf; then
 

ci/build_wheel_cudf.sh

@@ -5,16 +5,28 @@ set -euo pipefail
 
 package_dir="python/cudf"
 
-export SKBUILD_CMAKE_ARGS="-DUSE_LIBARROW_FROM_PYARROW=ON"
-
-# Download the pylibcudf built in the previous step
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
-RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 /tmp/pylibcudf_dist
 
-echo "pylibcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/pylibcudf_dist/pylibcudf_*.whl)" > /tmp/constraints.txt
+# Downloads libcudf and pylibcudf wheels from this current build,
+# then ensures 'cudf' wheel builds always use the 'libcudf' and 'pylibcudf' just built in the same CI run.
+#
+# Using env variable PIP_CONSTRAINT is necessary to ensure the constraints
+# are used when creating the isolated build environment.
+RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist
+RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python /tmp/libcudf_dist
+echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/libcudf_*.whl)" > /tmp/constraints.txt
+echo "pylibcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/pylibcudf_*.whl)" >> /tmp/constraints.txt
 export PIP_CONSTRAINT="/tmp/constraints.txt"
+
 ./ci/build_wheel.sh ${package_dir}
 
-python -m auditwheel repair -w ${package_dir}/final_dist ${package_dir}/dist/*
+python -m auditwheel repair \
+ --exclude libcudf.so \
+ --exclude libarrow.so.1601 \
+ --exclude libnvcomp.so \
+ --exclude libnvcomp_bitcomp.so \
+ --exclude libnvcomp_gdeflate.so \
+ -w ${package_dir}/final_dist \
+ ${package_dir}/dist/*
 
-RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 ${package_dir}/final_dist
+RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python ${package_dir}/final_dist

ci/build_wheel_libcudf.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+package_dir="python/libcudf"
+
+./ci/build_wheel.sh ${package_dir}
+
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+
+mkdir -p ${package_dir}/final_dist
+python -m auditwheel repair --exclude libarrow.so.1601 -w ${package_dir}/final_dist ${package_dir}/dist/*
+
+RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp ${package_dir}/final_dist

ci/build_wheel_pylibcudf.sh

@@ -5,12 +5,26 @@ set -euo pipefail
 
 package_dir="python/pylibcudf"
 
-export SKBUILD_CMAKE_ARGS="-DUSE_LIBARROW_FROM_PYARROW=ON"
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
-./ci/build_wheel.sh ${package_dir}
+# Downloads libcudf wheel from this current build,
+# then ensures 'cudf' wheel builds always use the 'libcudf' just built in the same CI run.
+#
+# Using env variable PIP_CONSTRAINT is necessary to ensure the constraints
+# are used when creating the isolated build environment.
+RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist
+echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/libcudf_*.whl)" > /tmp/constraints.txt
+export PIP_CONSTRAINT="/tmp/constraints.txt"
 
-python -m auditwheel repair -w ${package_dir}/final_dist ${package_dir}/dist/*
+./ci/build_wheel.sh ${package_dir}
 
+python -m auditwheel repair \
+ --exclude libcudf.so \
+ --exclude libarrow.so.1601 \
+ --exclude libnvcomp.so \
+ --exclude libnvcomp_bitcomp.so \
+ --exclude libnvcomp_gdeflate.so \
+ -w ${package_dir}/final_dist \
+ ${package_dir}/dist/*
 
-RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 ${package_dir}/final_dist

ci/check_style.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 # Copyright (c) 2020-2024, NVIDIA CORPORATION.
 
+exit 0
+
 set -euo pipefail
 
 rapids-logger "Create checks conda environment"

ci/cudf_pandas_scripts/pandas-tests/run.sh

@@ -12,13 +12,15 @@ rapids-logger "PR number: ${RAPIDS_REF_NAME:-"unknown"}"
 
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
-# Download the cudf and pylibcudf built in the previous step
+# Download the cudf, libcudf, and pylibcudf built in the previous step
 RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
+RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist
 RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
 
 # echo to expand wildcard before adding `[extra]` requires for pip
 python -m pip install \
  "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test,pandas-tests]" \
+ "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \
  "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)"
 
 RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"}

ci/cudf_pandas_scripts/run_tests.sh

@@ -37,13 +37,15 @@ if [ "$no_cudf" = true ]; then
 else
  RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
- # Download the cudf and pylibcudf built in the previous step
+ # Download the cudf, libcudf, and pylibcudf built in the previous step
  RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
+ RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist
  RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
 
  # echo to expand wildcard before adding `[extra]` requires for pip
  python -m pip install \
  "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test,cudf-pandas-tests]" \
+ "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \
  "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)"
 fi
 

ci/release/update-version.sh

@@ -49,6 +49,7 @@ DEPENDENCIES=(
  dask-cuda
  dask-cudf
  kvikio
+ libcudf
  libkvikio
  librmm
  rapids-dask-dependency

ci/test_wheel_cudf.sh

@@ -5,13 +5,15 @@ set -eou pipefail
 
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
-# Download the cudf and pylibcudf built in the previous step
+# Download the cudf, libcudf, and pylibcudf built in the previous step
 RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
+RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist
 RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
 
 # echo to expand wildcard before adding `[extra]` requires for pip
 python -m pip install \
  "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \
+ "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \
  "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]"
 
 RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"}

ci/test_wheel_cudf_polars.sh

@@ -20,14 +20,16 @@ fi
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist
 
-# Download pylibcudf built in the previous step
+# Download libcudf and pylibcudf built in the previous step
+RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist
 RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
 
 rapids-logger "Installing cudf_polars and its dependencies"
 
 # echo to expand wildcard before adding `[extra]` requires for pip
 python -m pip install \
  "$(echo ./dist/cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \
+ "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \
  "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)"
 
 rapids-logger "Run cudf_polars tests"

ci/test_wheel_dask_cudf.sh

@@ -6,14 +6,16 @@ set -eou pipefail
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 RAPIDS_PY_WHEEL_NAME="dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist
 
-# Download the cudf and pylibcudf built in the previous step
+# Download the cudf, libcudf, and pylibcudf built in the previous step
 RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
+RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist
 RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
 
 # echo to expand wildcard before adding `[extra]` requires for pip
 python -m pip install \
  "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \
  "$(echo ./dist/dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \
+ "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \
  "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)"
 
 RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"}