From a3de193f2a89e646a7f6e14cfa6d3f56eab7346c Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 8 Apr 2024 08:45:23 -0700 Subject: [PATCH 01/84] initial work on adding wheel build for libcudf --- .github/workflows/pr.yaml | 11 ++- build.sh | 16 +++- ci/build_wheel.sh | 2 + ci/build_wheel_cudf.sh | 14 +++- ci/build_wheel_libcudf.sh | 11 +++ ci/cudf_pandas_scripts/pandas-tests/run.sh | 5 +- ci/cudf_pandas_scripts/run_tests.sh | 5 +- ci/test_wheel_cudf.sh | 5 +- ci/test_wheel_dask_cudf.sh | 5 +- dependencies.yaml | 22 +++++ python/cudf/CMakeLists.txt | 84 +++++-------------- python/cudf/cudf/__init__.py | 10 +++ python/cudf/cudf/_lib/CMakeLists.txt | 2 + .../cudf/cudf/_lib/pylibcudf/CMakeLists.txt | 1 + python/cudf/pyproject.toml | 2 + python/libcudf/CMakeLists.txt | 37 ++++++++ python/libcudf/LICENSE | 1 + python/libcudf/README.md | 1 + python/libcudf/libcudf/VERSION | 1 + python/libcudf/libcudf/__init__.py | 16 ++++ python/libcudf/libcudf/_version.py | 23 +++++ python/libcudf/libcudf/load.py | 53 ++++++++++++ python/libcudf/pyproject.toml | 65 ++++++++++++++ 23 files changed, 312 insertions(+), 80 deletions(-) create mode 100755 ci/build_wheel_libcudf.sh create mode 100644 python/libcudf/CMakeLists.txt create mode 120000 python/libcudf/LICENSE create mode 120000 python/libcudf/README.md create mode 120000 python/libcudf/libcudf/VERSION create mode 100644 python/libcudf/libcudf/__init__.py create mode 100644 python/libcudf/libcudf/_version.py create mode 100644 python/libcudf/libcudf/load.py create mode 100644 python/libcudf/pyproject.toml diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f9d5976f1fe..e801d2f7f79 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -23,6 +23,7 @@ jobs: - static-configure - conda-notebook-tests - docs-build + - wheel-build-libcudf - wheel-build-cudf - wheel-tests-cudf - wheel-build-dask-cudf @@ -118,10 +119,18 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:latest" run_script: "ci/build_docs.sh" - wheel-build-cudf: + wheel-build-libcudf: needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 + with: + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) + build_type: pull-request + script: "ci/build_wheel_libcudf.sh" + wheel-build-cudf: + needs: [checks, wheel-build-libcudf] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: build_type: pull-request script: "ci/build_wheel_cudf.sh" diff --git a/build.sh b/build.sh index e5daf2f3451..a14d1ea85c4 100755 --- a/build.sh +++ b/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2019-2023, NVIDIA CORPORATION. +# Copyright (c) 2019-2024, NVIDIA CORPORATION. # cuDF build script @@ -17,12 +17,13 @@ ARGS=$* # script, and that this script resides in the repo dir! REPODIR=$(cd $(dirname $0); pwd) -VALIDARGS="clean libcudf cudf cudfjar dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz -v -g -n --pydevelop -l --allgpuarch --disable_nvtx --opensource_nvcomp --show_depr_warn --ptds -h --build_metrics --incl_cache_stats" -HELP="$0 [clean] [libcudf] [cudf] [cudfjar] [dask_cudf] [benchmarks] [tests] [libcudf_kafka] [cudf_kafka] [custreamz] [-v] [-g] [-n] [-h] [--cmake-args=\\\"\\\"] +VALIDARGS="clean libcudf cudf libcudfwheel cudfjar dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz -v -g -n --pydevelop -l --allgpuarch --disable_nvtx --opensource_nvcomp --show_depr_warn --ptds -h --build_metrics --incl_cache_stats" +HELP="$0 [clean] [libcudf] [cudf] [libcudfwheel] [cudfjar] [dask_cudf] [benchmarks] [tests] [libcudf_kafka] [cudf_kafka] [custreamz] [-v] [-g] [-n] [-h] [--cmake-args=\\\"\\\"] clean - remove all existing build artifacts and configuration (start over) libcudf - build the cudf C++ code only cudf - build the cudf Python package + libcudfwheel - build the cudf C++ code packaged as a python wheel package cudfjar - build cudf JAR with static libcudf using devtoolset toolchain dask_cudf - build the dask_cudf Python package benchmarks - build benchmarks @@ -333,7 +334,14 @@ if buildAll || hasArg libcudf; then fi fi -# Build and install the cudf Python package +if buildAll || hasArg libcudfwheel; then + + cd ${REPODIR}/python/libcudf + SKBUILD_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${INSTALL_PREFIX};-DCMAKE_LIBRARY_PATH=${LIBCUDF_BUILD_DIR};-DCMAKE_CUDA_ARCHITECTURES=${CUDF_CMAKE_CUDA_ARCHITECTURES};${EXTRA_CMAKE_ARGS}" \ + python ${PYTHON_ARGS_FOR_INSTALL} . +fi + +# Build and install the cudf Python packages if buildAll || hasArg cudf; then cd ${REPODIR}/python/cudf diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index c4b794e81f7..380ec3fe0ea 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -34,6 +34,8 @@ if ! rapids-is-release-build; then alpha_spec=',>=0.0.0a0' fi +sed -r -i "s/libcudf==(.*)\"/libcudf${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} + if [[ ${package_name} == "dask-cudf" ]]; then sed -r -i "s/cudf==(.*)\"/cudf${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} sed -r -i "s/dask-cuda==(.*)\"/dask-cuda==\1${alpha_spec}\"/g" ${pyproject_file} diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_cudf.sh index f0886a28fd9..2c609486a81 100755 --- a/ci/build_wheel_cudf.sh +++ b/ci/build_wheel_cudf.sh @@ -3,14 +3,20 @@ set -euo pipefail +package_name="cudf" package_dir="python/cudf" export SKBUILD_CMAKE_ARGS="-DUSE_LIBARROW_FROM_PYARROW=ON" -./ci/build_wheel.sh cudf ${package_dir} +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +# Downloads libcudf wheel from this current build, then points pip to it in PIP_FIND_LINKS below +RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist -python -m auditwheel repair -w ${package_dir}/final_dist ${package_dir}/dist/* +export PIP_FIND_LINKS="/tmp/libcudf_dist" +./ci/build_wheel.sh cudf ${package_dir} +cd ${package_dir} +mkdir -p final_dist +python -m auditwheel repair --exclude libcudf.so --exclude libarrow.so.1400 -w final_dist dist/* -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 ${package_dir}/final_dist +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh new file mode 100755 index 00000000000..eb1402c8a87 --- /dev/null +++ b/ci/build_wheel_libcudf.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +set -euo pipefail + +package_dir="python/libcudf" + +./ci/build_wheel.sh libcudf ${package_dir} + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp ${package_dir}/dist diff --git a/ci/cudf_pandas_scripts/pandas-tests/run.sh b/ci/cudf_pandas_scripts/pandas-tests/run.sh index abde5e5d160..562c2c12635 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/run.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/run.sh @@ -11,8 +11,9 @@ rapids-logger "Running Pandas tests using $PANDAS_TESTS_BRANCH branch and rapids rapids-logger "PR number: ${RAPIDS_REF_NAME:-"unknown"}" RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cudf-dep -python -m pip install $(ls ./local-cudf-dep/cudf*.whl)[test,pandas-tests] +RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-cudf-dep +RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./local-cudf-dep +python -m pip install --find-links $(pwd)/local-cudf-dep $(ls ./local-cudf-dep/cudf*.whl)[test,pandas-tests] RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index 78945d37f22..cb980ba9538 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -32,8 +32,9 @@ if [ "$no_cudf" = true ]; then echo "Skipping cudf install" else RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" - RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cudf-dep - python -m pip install $(ls ./local-cudf-dep/cudf*.whl)[test,cudf-pandas-tests] + RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-cudf-dep + RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./local-cudf-dep + python -m pip install --find-links $(pwd)/local-cudf-dep $(ls ./local-cudf-dep/cudf*.whl)[test,cudf-pandas-tests] fi python -m pytest -p cudf.pandas ./python/cudf/cudf_pandas_tests/ diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh index fdb61278d36..27db3767e40 100755 --- a/ci/test_wheel_cudf.sh +++ b/ci/test_wheel_cudf.sh @@ -4,10 +4,11 @@ set -eou pipefail RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist +RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist # echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install $(echo ./dist/cudf*.whl)[test] +python -m pip install --find-links $(pwd)/dist $(echo ./dist/cudf*.whl)[test] RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ diff --git a/ci/test_wheel_dask_cudf.sh b/ci/test_wheel_dask_cudf.sh index 2b20b9d9ce4..798ed8df062 100755 --- a/ci/test_wheel_dask_cudf.sh +++ b/ci/test_wheel_dask_cudf.sh @@ -7,8 +7,9 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist # Download the cudf built in the previous step -RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cudf-dep -python -m pip install --no-deps ./local-cudf-dep/cudf*.whl +RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-cudf-dep +RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./local-cudf-dep +python -m pip install --no-deps --find-links $(pwd)/local-cudf-dep ./local-cudf-dep/cudf*.whl # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/dask_cudf*.whl)[test] diff --git a/dependencies.yaml b/dependencies.yaml index 14c698000cb..1506ccef87e 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -21,6 +21,7 @@ files: - py_version - run_common - run_cudf + - run_libcudf - run_dask_cudf - run_custreamz - test_cpp @@ -75,6 +76,18 @@ files: - docs - libarrow_run - py_version + # This is the shared library, bundled as a wheel. It is meant to be consumed by the wrapper. + py_build_libcudf: + output: pyproject + pyproject_dir: python/libcudf + extras: + table: build-system + includes: + - build_base + - build_cpp + - build_python_common + - build_python_cudf + # This is the wrapper that gets used in Python, not the shared library wheel py_build_cudf: output: pyproject pyproject_dir: python/cudf @@ -84,6 +97,7 @@ files: - build_base - build_python_common - build_python_cudf + - run_libcudf py_run_cudf: output: pyproject pyproject_dir: python/cudf @@ -94,6 +108,7 @@ files: - run_cudf - pyarrow_run - depends_on_cupy + - run_libcudf py_test_cudf: output: pyproject pyproject_dir: python/cudf @@ -336,6 +351,13 @@ dependencies: # Allow runtime version to float up to minor version # Disallow pyarrow 14.0.0 due to a CVE - pyarrow>=14.0.1,<15.0.0a0 + run_libcudf: + common: + # TODO: Currently this is a hack for devcontainers. Need to figure out the best solution. + #- output_types: [requirements, pyproject] + - output_types: [pyproject] + packages: + - libcudf==24.6.* cuda_version: specific: - output_types: conda diff --git a/python/cudf/CMakeLists.txt b/python/cudf/CMakeLists.txt index ecadbf5cbbc..6c11c4c0fe0 100644 --- a/python/cudf/CMakeLists.txt +++ b/python/cudf/CMakeLists.txt @@ -24,72 +24,15 @@ project( LANGUAGES CXX CUDA ) -option(FIND_CUDF_CPP "Search for existing CUDF C++ installations before defaulting to local files" - OFF -) -option(USE_LIBARROW_FROM_PYARROW "Only use the libarrow contained in pyarrow" OFF) -mark_as_advanced(USE_LIBARROW_FROM_PYARROW) - -# Find Python early so that later commands can use it -find_package(Python 3.9 REQUIRED COMPONENTS Interpreter) - -# If the user requested it we attempt to find CUDF. -if(FIND_CUDF_CPP) - include(rapids-cpm) - include(rapids-export) - include(rapids-find) - rapids_cpm_init() - - if(USE_LIBARROW_FROM_PYARROW) - # We need to find arrow before libcudf since libcudf requires it but doesn't bundle arrow - # libraries. These variables have no effect because we are always searching for arrow via - # pyarrow, but they must be set as they are required arguments to the function in - # get_arrow.cmake. - set(CUDF_USE_ARROW_STATIC OFF) - set(CUDF_ENABLE_ARROW_S3 OFF) - set(CUDF_ENABLE_ARROW_ORC OFF) - set(CUDF_ENABLE_ARROW_PYTHON OFF) - set(CUDF_ENABLE_ARROW_PARQUET OFF) - include(../../cpp/cmake/thirdparty/get_arrow.cmake) - endif() +find_package(cudf "${RAPIDS_VERSION}" REQUIRED) - find_package(cudf "${RAPIDS_VERSION}" REQUIRED) - - # an installed version of libcudf doesn't provide the dlpack headers so we need to download dlpack - # for the interop.pyx - include(../../cpp/cmake/thirdparty/get_dlpack.cmake) -else() - set(cudf_FOUND OFF) -endif() +# an installed version of libcudf doesn't provide the dlpack headers so we need to download dlpack +# for the interop.pyx +include(rapids-cpm) +rapids_cpm_init() +include(../../cpp/cmake/thirdparty/get_dlpack.cmake) include(rapids-cython-core) - -if(NOT cudf_FOUND) - set(BUILD_TESTS OFF) - set(BUILD_BENCHMARKS OFF) - set(CUDF_BUILD_TESTUTIL OFF) - set(CUDF_BUILD_STREAMS_TEST_UTIL OFF) - set(CUDA_STATIC_RUNTIME ON) - - add_subdirectory(../../cpp cudf-cpp EXCLUDE_FROM_ALL) - - # libcudf targets are excluded by default above via EXCLUDE_FROM_ALL to remove extraneous - # components like headers from libcudacxx, but we do need the libraries. However, we want to - # control where they are installed to. Since there are multiple subpackages of cudf._lib that - # require access to libcudf, we place the library and all its dependent artifacts in the cudf - # directory as a single source of truth and modify the other rpaths appropriately. - set(cython_lib_dir cudf) - include(cmake/Modules/WheelHelpers.cmake) - # TODO: This install is currently overzealous. We should only install the libraries that are - # downloaded by CPM during the build, not libraries that were found on the system. However, in - # practice right this would only be a problem is if libcudf was not found but some of the - # dependencies were, and we have no real use cases where that happens. - install_aliased_imported_targets( - TARGETS cudf arrow_shared nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp - DESTINATION ${cython_lib_dir} - ) -endif() - rapids_cython_init() include(cmake/Modules/LinkPyarrowHeaders.cmake) @@ -99,3 +42,18 @@ add_subdirectory(udf_cpp) if(DEFINED cython_lib_dir) rapids_cython_add_rpath_entries(TARGET cudf PATHS "${cython_lib_dir}") endif() + +# libcudf targets are excluded by default above via EXCLUDE_FROM_ALL to remove extraneous components +# like headers from libcudacxx, but we do need the libraries. However, we want to control where they +# are installed to. Since there are multiple subpackages of cudf._lib that require access to +# libcudf, we place the library and all its dependent artifacts in the cudf directory as a single +# source of truth and modify the other rpaths appropriately. +include(cmake/Modules/WheelHelpers.cmake) +# TODO: This install is currently overzealous. We should only install the libraries that are +# downloaded by CPM during the build, not libraries that were found on the system. However, in +# practice right this would only be a problem is if libcudf was not found but some of the +# dependencies were, and we have no real use cases where that happens. +install_aliased_imported_targets( + TARGETS cudf arrow_shared nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp + DESTINATION ${cython_lib_dir} +) diff --git a/python/cudf/cudf/__init__.py b/python/cudf/cudf/__init__.py index e14815a1b0d..8e10c0a2890 100644 --- a/python/cudf/cudf/__init__.py +++ b/python/cudf/cudf/__init__.py @@ -1,5 +1,15 @@ # Copyright (c) 2018-2024, NVIDIA CORPORATION. +# If libcudf was installed as a wheel, we must request it to load the library symbols. +# Otherwise, we assume that the library was installed in a system path that ld can find. +try: + import libcudf +except ModuleNotFoundError: + pass +else: + libcudf.load_library() + del libcudf + # _setup_numba _must be called before numba.cuda is imported, because # it sets the numba config variable responsible for enabling # Minor Version Compatibility. Setting it after importing numba.cuda has no effect. diff --git a/python/cudf/cudf/_lib/CMakeLists.txt b/python/cudf/cudf/_lib/CMakeLists.txt index 07f334fdc12..1ed6e47f99f 100644 --- a/python/cudf/cudf/_lib/CMakeLists.txt +++ b/python/cudf/cudf/_lib/CMakeLists.txt @@ -68,6 +68,8 @@ target_link_libraries(strings_udf PUBLIC cudf_strings_udf) set(targets_using_arrow_headers interop avro csv orc json parquet) link_to_pyarrow_headers("${targets_using_arrow_headers}") +target_include_directories(interop PUBLIC "$") + add_subdirectory(cpp) add_subdirectory(io) add_subdirectory(nvtext) diff --git a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt index c2b7cb7ca3d..cb255c66978 100644 --- a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt +++ b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt @@ -44,5 +44,6 @@ rapids_cython_create_modules( LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX pylibcudf_ ASSOCIATED_TARGETS cudf ) link_to_pyarrow_headers(pylibcudf_interop) +target_include_directories(pylibcudf_interop PUBLIC "$") add_subdirectory(strings) diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index fc3a243572f..9b659e3b04f 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -5,6 +5,7 @@ build-backend = "scikit_build_core.build" requires = [ "cmake>=3.26.4", "cython>=3.0.3", + "libcudf==24.6.*", "ninja", "numpy==1.23.*", "pyarrow==14.0.2.*", @@ -28,6 +29,7 @@ dependencies = [ "cuda-python>=11.7.1,<12.0a0", "cupy-cuda11x>=12.0.0", "fsspec>=0.6.0", + "libcudf==24.6.*", "numba>=0.57", "numpy>=1.23,<2.0a0", "nvtx>=0.2.1", diff --git a/python/libcudf/CMakeLists.txt b/python/libcudf/CMakeLists.txt new file mode 100644 index 00000000000..5ccfd8f4181 --- /dev/null +++ b/python/libcudf/CMakeLists.txt @@ -0,0 +1,37 @@ +# ============================================================================= +# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) + +include(../../rapids_config.cmake) + +project( + libcudf-python + VERSION "${RAPIDS_VERSION}" + LANGUAGES CXX +) + +# For wheels, this should always be true +set(USE_LIBARROW_FROM_PYARROW ON) + +# Find Python early so that later commands can use it +find_package(Python 3.9 REQUIRED COMPONENTS Interpreter) + +set(BUILD_TESTS OFF) +set(BUILD_BENCHMARKS OFF) +set(CUDF_BUILD_TESTUTIL OFF) +set(CUDF_BUILD_STREAMS_TEST_UTIL OFF) +set(CUDA_STATIC_RUNTIME ON) + +add_subdirectory(../../cpp cudf-cpp) diff --git a/python/libcudf/LICENSE b/python/libcudf/LICENSE new file mode 120000 index 00000000000..30cff7403da --- /dev/null +++ b/python/libcudf/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/python/libcudf/README.md b/python/libcudf/README.md new file mode 120000 index 00000000000..fe840054137 --- /dev/null +++ b/python/libcudf/README.md @@ -0,0 +1 @@ +../../README.md \ No newline at end of file diff --git a/python/libcudf/libcudf/VERSION b/python/libcudf/libcudf/VERSION new file mode 120000 index 00000000000..d62dc733efd --- /dev/null +++ b/python/libcudf/libcudf/VERSION @@ -0,0 +1 @@ +../../../VERSION \ No newline at end of file diff --git a/python/libcudf/libcudf/__init__.py b/python/libcudf/libcudf/__init__.py new file mode 100644 index 00000000000..10c476cbe89 --- /dev/null +++ b/python/libcudf/libcudf/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from libcudf._version import __git_commit__, __version__ +from libcudf.load import load_library diff --git a/python/libcudf/libcudf/_version.py b/python/libcudf/libcudf/_version.py new file mode 100644 index 00000000000..da6b96c163e --- /dev/null +++ b/python/libcudf/libcudf/_version.py @@ -0,0 +1,23 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.resources + +__version__ = ( + importlib.resources.files("libcudf") + .joinpath("VERSION") + .read_text() + .strip() +) +__git_commit__ = "" diff --git a/python/libcudf/libcudf/load.py b/python/libcudf/libcudf/load.py new file mode 100644 index 00000000000..11bb1522942 --- /dev/null +++ b/python/libcudf/libcudf/load.py @@ -0,0 +1,53 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import ctypes +import os + + +def load_library(): + # this is loading the libarrow shared library + import pyarrow # noqa: F401 + + # Dynamically load libcudf.so. Prefer a system library if one is present to + # avoid clobbering symbols that other packages might expect, but if no + # other library is present use the one in the wheel. + try: + libcudf_lib = ctypes.CDLL("libcudf.so", ctypes.RTLD_GLOBAL) + except OSError: + # load the nvcomp libraries from lib first, so that libcudf has them when we load it from lib64 + # TODO: if we put both of these in the same folder, I don't think we'd need this. + for nvcomp_lib in ( + "libnvcomp.so", + "libnvcomp_gdeflate.so", + "libnvcomp_bitcomp.so", + ): + _loaded_lib = ctypes.CDLL( + # TODO: Do we always know it will be lib64? Should we consider + # finding a way for CMake to export the path for us to find here? + os.path.join(os.path.dirname(__file__), "lib", nvcomp_lib), + ctypes.RTLD_GLOBAL, + ) + libcudf_lib = ctypes.CDLL( + # TODO: Do we always know it will be lib64? Should we consider + # finding a way for CMake to export the path for us to find here? + os.path.join(os.path.dirname(__file__), "lib64", "libcudf.so"), + ctypes.RTLD_GLOBAL, + ) + + # The caller almost never needs to do anything with this library, but no + # harm in offering the option since this object at least provides a handle + # to inspect where libcudf was loaded from. + return libcudf_lib diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml new file mode 100644 index 00000000000..4b2f3c46029 --- /dev/null +++ b/python/libcudf/pyproject.toml @@ -0,0 +1,65 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[build-system] +build-backend = "scikit_build_core.build" +requires = [ + "cmake>=3.26.4", + "cython>=3.0.3", + "ninja", + "numpy==1.23.*", + "pyarrow==14.0.2.*", + "rmm==24.6.*", + "scikit-build-core[pyproject]>=0.7.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project] +name = "libcudf" +dynamic = ["version"] +description = "cuDF - GPU Dataframe" +readme = { file = "README.md", content-type = "text/markdown" } +authors = [ + { name = "NVIDIA Corporation" }, +] +license = { text = "Apache 2.0" } +requires-python = ">=3.9" +classifiers = [ + "Intended Audience :: Developers", + "Topic :: Database", + "Topic :: Scientific/Engineering", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: C++", + "Environment :: GPU :: NVIDIA CUDA", +] + +[project.urls] +Homepage = "https://github.com/rapidsai/cudf" + +[project.entry-points."cmake.prefix"] +libcudf = "libcudf" + +[tool.scikit-build] +build-dir = "build/{wheel_tag}" +cmake.build-type = "Release" +cmake.minimum-version = "3.26.4" +ninja.make-fallback = true +sdist.reproducible = true +wheel.packages = ["libcudf"] +wheel.install-dir = "libcudf" +wheel.py-api = "py3" + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "libcudf/VERSION" +regex = "(?P.*)" From b43ad02dbda743ed0ece0d7ccb28e392adcd625d Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 26 Apr 2024 11:34:43 -0500 Subject: [PATCH 02/84] add auditwheel to libcudf wheels --- ci/build_wheel_libcudf.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh index eb1402c8a87..351b9d7fc40 100755 --- a/ci/build_wheel_libcudf.sh +++ b/ci/build_wheel_libcudf.sh @@ -8,4 +8,9 @@ package_dir="python/libcudf" ./ci/build_wheel.sh libcudf ${package_dir} RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp ${package_dir}/dist + +cd ${package_dir} +mkdir -p final_dist +python -m auditwheel repair --exclude libarrow.so.1400 -w final_dist dist/* + +RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp ${package_dir}/final_dist From d7dc523e9b7b703b01e872640a6fb96178a7a1d2 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 29 Apr 2024 13:44:21 -0500 Subject: [PATCH 03/84] check vyas' nvcomp rapids-cmake fix --- ci/build_wheel_libcudf.sh | 5 ++--- python/libcudf/CMakeLists.txt | 3 +++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh index 351b9d7fc40..5d96267b628 100755 --- a/ci/build_wheel_libcudf.sh +++ b/ci/build_wheel_libcudf.sh @@ -9,8 +9,7 @@ package_dir="python/libcudf" RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -cd ${package_dir} -mkdir -p final_dist -python -m auditwheel repair --exclude libarrow.so.1400 -w final_dist dist/* +mkdir -p ${package_dir}/final_dist +python -m auditwheel repair --exclude libarrow.so.1400 -w ${package_dir}/final_dist ${package_dir}/dist/* RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp ${package_dir}/final_dist diff --git a/python/libcudf/CMakeLists.txt b/python/libcudf/CMakeLists.txt index 5ccfd8f4181..93ad7156ab9 100644 --- a/python/libcudf/CMakeLists.txt +++ b/python/libcudf/CMakeLists.txt @@ -16,6 +16,9 @@ cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) include(../../rapids_config.cmake) +set(rapids-cmake-repo "vyasr") +set(rapids-cmake-branch "fix/nvcomp_lib_dir") + project( libcudf-python VERSION "${RAPIDS_VERSION}" From 0bb4efe5f3f5a03b0dcd647d57de60900f5d9579 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 29 Apr 2024 14:31:37 -0500 Subject: [PATCH 04/84] try to get rid of lib64 folder for wheel build --- python/libcudf/CMakeLists.txt | 2 ++ python/libcudf/libcudf/load.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/python/libcudf/CMakeLists.txt b/python/libcudf/CMakeLists.txt index 93ad7156ab9..9fb7ecaf6b0 100644 --- a/python/libcudf/CMakeLists.txt +++ b/python/libcudf/CMakeLists.txt @@ -37,4 +37,6 @@ set(CUDF_BUILD_TESTUTIL OFF) set(CUDF_BUILD_STREAMS_TEST_UTIL OFF) set(CUDA_STATIC_RUNTIME ON) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) + add_subdirectory(../../cpp cudf-cpp) diff --git a/python/libcudf/libcudf/load.py b/python/libcudf/libcudf/load.py index 11bb1522942..1453ba83d35 100644 --- a/python/libcudf/libcudf/load.py +++ b/python/libcudf/libcudf/load.py @@ -43,7 +43,7 @@ def load_library(): libcudf_lib = ctypes.CDLL( # TODO: Do we always know it will be lib64? Should we consider # finding a way for CMake to export the path for us to find here? - os.path.join(os.path.dirname(__file__), "lib64", "libcudf.so"), + os.path.join(os.path.dirname(__file__), "lib", "libcudf.so"), ctypes.RTLD_GLOBAL, ) From 1443f8b7b3744138dca5f8330a4151a38e848ecd Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 29 Apr 2024 14:59:12 -0500 Subject: [PATCH 05/84] fix location of rapids-cmake customization --- python/libcudf/CMakeLists.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/libcudf/CMakeLists.txt b/python/libcudf/CMakeLists.txt index 9fb7ecaf6b0..b59d0d57535 100644 --- a/python/libcudf/CMakeLists.txt +++ b/python/libcudf/CMakeLists.txt @@ -14,11 +14,12 @@ cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) -include(../../rapids_config.cmake) -set(rapids-cmake-repo "vyasr") +set(rapids-cmake-repo "vyasr/rapids-cmake") set(rapids-cmake-branch "fix/nvcomp_lib_dir") +include(../../rapids_config.cmake) + project( libcudf-python VERSION "${RAPIDS_VERSION}" From 4ccef6e261a0083a4be202b3e57cff989ba17b55 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 29 Apr 2024 16:37:45 -0500 Subject: [PATCH 06/84] whitespace =( --- python/libcudf/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/python/libcudf/CMakeLists.txt b/python/libcudf/CMakeLists.txt index b59d0d57535..c7a3cf3d1ef 100644 --- a/python/libcudf/CMakeLists.txt +++ b/python/libcudf/CMakeLists.txt @@ -14,7 +14,6 @@ cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) - set(rapids-cmake-repo "vyasr/rapids-cmake") set(rapids-cmake-branch "fix/nvcomp_lib_dir") From a2e0ca05daa59b51a8f41ff05a360202a4906354 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 30 Apr 2024 13:29:55 -0500 Subject: [PATCH 07/84] renaming wheel workflows for conformance with conda --- .github/workflows/pr.yaml | 26 +++++++++---------- ...ld_wheel_libcudf.sh => build_wheel_cpp.sh} | 0 ...ld_wheel_cudf.sh => build_wheel_python.sh} | 0 python/libcudf/libcudf/load.py | 4 +-- 4 files changed, 15 insertions(+), 15 deletions(-) rename ci/{build_wheel_libcudf.sh => build_wheel_cpp.sh} (100%) rename ci/{build_wheel_cudf.sh => build_wheel_python.sh} (100%) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e801d2f7f79..9c128566b89 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -23,9 +23,9 @@ jobs: - static-configure - conda-notebook-tests - docs-build - - wheel-build-libcudf - - wheel-build-cudf - - wheel-tests-cudf + - wheel-build-cpp + - wheel-build-python + - wheel-tests-python - wheel-build-dask-cudf - wheel-tests-dask-cudf - devcontainer @@ -119,30 +119,30 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:latest" run_script: "ci/build_docs.sh" - wheel-build-libcudf: + wheel-build-cpp: needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: pull-request - script: "ci/build_wheel_libcudf.sh" - wheel-build-cudf: - needs: [checks, wheel-build-libcudf] + script: "ci/build_wheel_cpp.sh" + wheel-build-python: + needs: [checks, wheel-build-cpp] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: build_type: pull-request - script: "ci/build_wheel_cudf.sh" - wheel-tests-cudf: - needs: wheel-build-cudf + script: "ci/build_wheel_python.sh" + wheel-tests-python: + needs: wheel-build-python secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 with: build_type: pull-request script: ci/test_wheel_cudf.sh wheel-build-dask-cudf: - needs: wheel-build-cudf + needs: wheel-build-python secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: @@ -170,7 +170,7 @@ jobs: build-all -DBUILD_BENCHMARKS=ON --verbose; sccache -s; unit-tests-cudf-pandas: - needs: wheel-build-cudf + needs: wheel-build-python secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 with: @@ -179,7 +179,7 @@ jobs: script: ci/cudf_pandas_scripts/run_tests.sh pandas-tests: # run the Pandas unit tests using PR branch - needs: wheel-build-cudf + needs: wheel-build-python secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 with: diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_cpp.sh similarity index 100% rename from ci/build_wheel_libcudf.sh rename to ci/build_wheel_cpp.sh diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_python.sh similarity index 100% rename from ci/build_wheel_cudf.sh rename to ci/build_wheel_python.sh diff --git a/python/libcudf/libcudf/load.py b/python/libcudf/libcudf/load.py index 1453ba83d35..ce7544bb202 100644 --- a/python/libcudf/libcudf/load.py +++ b/python/libcudf/libcudf/load.py @@ -37,13 +37,13 @@ def load_library(): _loaded_lib = ctypes.CDLL( # TODO: Do we always know it will be lib64? Should we consider # finding a way for CMake to export the path for us to find here? - os.path.join(os.path.dirname(__file__), "lib", nvcomp_lib), + os.path.join(os.path.dirname(__file__), "lib64", nvcomp_lib), ctypes.RTLD_GLOBAL, ) libcudf_lib = ctypes.CDLL( # TODO: Do we always know it will be lib64? Should we consider # finding a way for CMake to export the path for us to find here? - os.path.join(os.path.dirname(__file__), "lib", "libcudf.so"), + os.path.join(os.path.dirname(__file__), "lib64", "libcudf.so"), ctypes.RTLD_GLOBAL, ) From ef4b2c991ab51d20533d88587f5440d6cacdb7c3 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 1 May 2024 19:58:04 +0000 Subject: [PATCH 08/84] Try hiding logger symbols --- cpp/CMakeLists.txt | 5 +++++ cpp/include/cudf/utilities/logger.hpp | 1 + 2 files changed, 6 insertions(+) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 60d0094efac..22b216f9b40 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -678,6 +678,11 @@ set_source_files_properties( src/transform/transform.cpp PROPERTIES COMPILE_DEFINITIONS "_FILE_OFFSET_BITS=64" ) +set_source_files_properties( + src/utilities/logger.cpp + PROPERTIES COMPILE_OPTIONS "-fvisibility=hidden;-fvisibility-inlines-hidden" +) + set_property( SOURCE src/io/parquet/writer_impl.cu diff --git a/cpp/include/cudf/utilities/logger.hpp b/cpp/include/cudf/utilities/logger.hpp index a39df064f44..d4729377605 100644 --- a/cpp/include/cudf/utilities/logger.hpp +++ b/cpp/include/cudf/utilities/logger.hpp @@ -41,6 +41,7 @@ namespace cudf { * * @return spdlog::logger& The logger. */ +__attribute__((visibility("default"))) spdlog::logger& logger(); } // namespace cudf From ff0ac5a54647e108aa6191df736e061bb17568ae Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 1 May 2024 20:21:52 +0000 Subject: [PATCH 09/84] Style --- cpp/CMakeLists.txt | 5 ++--- cpp/include/cudf/utilities/logger.hpp | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 22b216f9b40..3db8272de2c 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -679,11 +679,10 @@ set_source_files_properties( PROPERTIES COMPILE_DEFINITIONS "_FILE_OFFSET_BITS=64" ) set_source_files_properties( - src/utilities/logger.cpp - PROPERTIES COMPILE_OPTIONS "-fvisibility=hidden;-fvisibility-inlines-hidden" + src/utilities/logger.cpp PROPERTIES COMPILE_OPTIONS + "-fvisibility=hidden;-fvisibility-inlines-hidden" ) - set_property( SOURCE src/io/parquet/writer_impl.cu APPEND diff --git a/cpp/include/cudf/utilities/logger.hpp b/cpp/include/cudf/utilities/logger.hpp index d4729377605..6c001c075b4 100644 --- a/cpp/include/cudf/utilities/logger.hpp +++ b/cpp/include/cudf/utilities/logger.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,7 +41,6 @@ namespace cudf { * * @return spdlog::logger& The logger. */ -__attribute__((visibility("default"))) -spdlog::logger& logger(); +__attribute__((visibility("default"))) spdlog::logger& logger(); } // namespace cudf From 1cdaef63534b1e4fcb18bcac6fe5591ecd76b03a Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 1 May 2024 20:42:19 +0000 Subject: [PATCH 10/84] Ignore doxygen, which is confused by the attribute --- cpp/include/cudf/utilities/logger.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/include/cudf/utilities/logger.hpp b/cpp/include/cudf/utilities/logger.hpp index 6c001c075b4..5ca4a603079 100644 --- a/cpp/include/cudf/utilities/logger.hpp +++ b/cpp/include/cudf/utilities/logger.hpp @@ -41,6 +41,8 @@ namespace cudf { * * @return spdlog::logger& The logger. */ +/// \cond DOXYGEN_IGNORE __attribute__((visibility("default"))) spdlog::logger& logger(); +/// \endcond } // namespace cudf From 520c122cb98a8ecd7cd970012b1e27a1cbbe9d8b Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 1 May 2024 22:50:46 +0000 Subject: [PATCH 11/84] Add comment --- cpp/CMakeLists.txt | 7 +++++++ cpp/include/cudf/utilities/logger.hpp | 3 +++ 2 files changed, 10 insertions(+) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 3db8272de2c..0b12341c3e3 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -678,6 +678,13 @@ set_source_files_properties( src/transform/transform.cpp PROPERTIES COMPILE_DEFINITIONS "_FILE_OFFSET_BITS=64" ) +# We need to hide symbols in the logger TU by default because those symbols +# include spdlog symbols that are compiled in because we use it as a +# header-only library. This is problematic because other libraries that also do +# the same thing will have their own copies, and they will all be weak symbols +# so which one is used is undefined. Marking these symbols as hidden makes them +# local and ensures that internal calls to spdlog functions correctly use the +# local copy. set_source_files_properties( src/utilities/logger.cpp PROPERTIES COMPILE_OPTIONS "-fvisibility=hidden;-fvisibility-inlines-hidden" diff --git a/cpp/include/cudf/utilities/logger.hpp b/cpp/include/cudf/utilities/logger.hpp index 5ca4a603079..333aea5b5e2 100644 --- a/cpp/include/cudf/utilities/logger.hpp +++ b/cpp/include/cudf/utilities/logger.hpp @@ -41,7 +41,10 @@ namespace cudf { * * @return spdlog::logger& The logger. */ +// This is a workaround for Doxygen not being able to parse the `visibility` attribute /// \cond DOXYGEN_IGNORE +// We need this symbol to be visible because this TU is marked as having all +// symbols hidden by default in cpp/CMakelists.txt. __attribute__((visibility("default"))) spdlog::logger& logger(); /// \endcond From 42f0b4812eb88c5cf208182521aef214e772b774 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 1 May 2024 23:14:18 +0000 Subject: [PATCH 12/84] Fix formatting --- cpp/CMakeLists.txt | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 0b12341c3e3..4e0d91af321 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -678,13 +678,11 @@ set_source_files_properties( src/transform/transform.cpp PROPERTIES COMPILE_DEFINITIONS "_FILE_OFFSET_BITS=64" ) -# We need to hide symbols in the logger TU by default because those symbols -# include spdlog symbols that are compiled in because we use it as a -# header-only library. This is problematic because other libraries that also do -# the same thing will have their own copies, and they will all be weak symbols -# so which one is used is undefined. Marking these symbols as hidden makes them -# local and ensures that internal calls to spdlog functions correctly use the -# local copy. +# We need to hide symbols in the logger TU by default because those symbols include spdlog symbols +# that are compiled in because we use it as a header-only library. This is problematic because other +# libraries that also do the same thing will have their own copies, and they will all be weak +# symbols so which one is used is undefined. Marking these symbols as hidden makes them local and +# ensures that internal calls to spdlog functions correctly use the local copy. set_source_files_properties( src/utilities/logger.cpp PROPERTIES COMPILE_OPTIONS "-fvisibility=hidden;-fvisibility-inlines-hidden" From b8ae5c3d7a5f5ef2693bf952b9aadf10547f30ae Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 2 May 2024 23:56:34 +0000 Subject: [PATCH 13/84] Update library loading logic for the new nvcomp layout and make build compatible with devcontainers --- python/libcudf/CMakeLists.txt | 10 ++++++++++ python/libcudf/libcudf/load.py | 36 +++++++++++++++++----------------- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/python/libcudf/CMakeLists.txt b/python/libcudf/CMakeLists.txt index c7a3cf3d1ef..78a49b553e9 100644 --- a/python/libcudf/CMakeLists.txt +++ b/python/libcudf/CMakeLists.txt @@ -25,6 +25,16 @@ project( LANGUAGES CXX ) +# Check if cudf is already available. If so, it is the user's responsibility to ensure that the +# CMake package is also available at build time of the Python cudf package. +find_package(cudf "${RAPIDS_VERSION}") + +if(cudf_FOUND) + return() +endif() + +unset(cudf_FOUND) + # For wheels, this should always be true set(USE_LIBARROW_FROM_PYARROW ON) diff --git a/python/libcudf/libcudf/load.py b/python/libcudf/libcudf/load.py index ce7544bb202..91723553e1a 100644 --- a/python/libcudf/libcudf/load.py +++ b/python/libcudf/libcudf/load.py @@ -18,34 +18,34 @@ def load_library(): - # this is loading the libarrow shared library + # This is loading the libarrow shared library in situations where it comes from the + # pyarrow package (i.e. when installed as a wheel). import pyarrow # noqa: F401 # Dynamically load libcudf.so. Prefer a system library if one is present to # avoid clobbering symbols that other packages might expect, but if no # other library is present use the one in the wheel. + libcudf_lib = None try: libcudf_lib = ctypes.CDLL("libcudf.so", ctypes.RTLD_GLOBAL) except OSError: - # load the nvcomp libraries from lib first, so that libcudf has them when we load it from lib64 - # TODO: if we put both of these in the same folder, I don't think we'd need this. - for nvcomp_lib in ( - "libnvcomp.so", - "libnvcomp_gdeflate.so", - "libnvcomp_bitcomp.so", - ): - _loaded_lib = ctypes.CDLL( - # TODO: Do we always know it will be lib64? Should we consider - # finding a way for CMake to export the path for us to find here? - os.path.join(os.path.dirname(__file__), "lib64", nvcomp_lib), + this_dir = os.path.dirname(__file__) + lib_dir = os.path.join(this_dir, "lib") + lib64_dir = os.path.join(this_dir, "lib64") + for real_lib_dir in (lib_dir, lib64_dir): + if os.path.isdir(real_lib_dir): + break + else: + real_lib_dir = None + + # If neither directory was found in the wheel, we assume we are in an + # environment where the C++ library is already installed somewhere else and the + # CMake build of the libcudf Python package was a no-op. + if real_lib_dir is not None: + libcudf_lib = ctypes.CDLL( + os.path.join(real_lib_dir, "libcudf.so"), ctypes.RTLD_GLOBAL, ) - libcudf_lib = ctypes.CDLL( - # TODO: Do we always know it will be lib64? Should we consider - # finding a way for CMake to export the path for us to find here? - os.path.join(os.path.dirname(__file__), "lib64", "libcudf.so"), - ctypes.RTLD_GLOBAL, - ) # The caller almost never needs to do anything with this library, but no # harm in offering the option since this object at least provides a handle From ccf28f2aa97d1df7788c7358cd52fd15105ca3e9 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 3 May 2024 00:46:16 +0000 Subject: [PATCH 14/84] Fix loading logic --- python/libcudf/libcudf/load.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/python/libcudf/libcudf/load.py b/python/libcudf/libcudf/load.py index 91723553e1a..a2a49ff7f4b 100644 --- a/python/libcudf/libcudf/load.py +++ b/python/libcudf/libcudf/load.py @@ -29,23 +29,13 @@ def load_library(): try: libcudf_lib = ctypes.CDLL("libcudf.so", ctypes.RTLD_GLOBAL) except OSError: - this_dir = os.path.dirname(__file__) - lib_dir = os.path.join(this_dir, "lib") - lib64_dir = os.path.join(this_dir, "lib64") - for real_lib_dir in (lib_dir, lib64_dir): - if os.path.isdir(real_lib_dir): - break - else: - real_lib_dir = None - - # If neither directory was found in the wheel, we assume we are in an + # If neither of these directories contain the library, we assume we are in an # environment where the C++ library is already installed somewhere else and the # CMake build of the libcudf Python package was a no-op. - if real_lib_dir is not None: - libcudf_lib = ctypes.CDLL( - os.path.join(real_lib_dir, "libcudf.so"), - ctypes.RTLD_GLOBAL, - ) + for lib_dir in ("lib", "lib64"): + if os.path.isfile(lib := os.path.join(lib_dir, "libcudf.so")): + libcudf_lib = ctypes.CDLL(lib, ctypes.RTLD_GLOBAL) + break # The caller almost never needs to do anything with this library, but no # harm in offering the option since this object at least provides a handle From 466cacb2bf368e28fd8795202e20fb5fe4adc3b6 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 3 May 2024 01:46:17 +0000 Subject: [PATCH 15/84] Fix paths again --- python/libcudf/libcudf/load.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/python/libcudf/libcudf/load.py b/python/libcudf/libcudf/load.py index a2a49ff7f4b..00f5e046538 100644 --- a/python/libcudf/libcudf/load.py +++ b/python/libcudf/libcudf/load.py @@ -31,9 +31,16 @@ def load_library(): except OSError: # If neither of these directories contain the library, we assume we are in an # environment where the C++ library is already installed somewhere else and the - # CMake build of the libcudf Python package was a no-op. + # CMake build of the libcudf Python package was a no-op. Note that this approach + # won't work for real editable installs of the libcudf package, but that's not a + # use case I think we need to support. scikit-build-core has limited support for + # importlib.resources so there isn't a clean way to support that case yet. for lib_dir in ("lib", "lib64"): - if os.path.isfile(lib := os.path.join(lib_dir, "libcudf.so")): + if os.path.isfile( + lib := os.path.join( + os.path.dirname(__file__), lib_dir, "libcudf.so" + ) + ): libcudf_lib = ctypes.CDLL(lib, ctypes.RTLD_GLOBAL) break From 97f4f958cd9a2bd4d84db2b6e21e2ed7d0b24058 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 24 Jun 2024 16:19:09 -0500 Subject: [PATCH 16/84] fix calls to build_wheel.sh in cpp and python scripts --- ci/build_wheel_cpp.sh | 2 +- ci/build_wheel_python.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 5d96267b628..0f8dd2208f9 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -5,7 +5,7 @@ set -euo pipefail package_dir="python/libcudf" -./ci/build_wheel.sh libcudf ${package_dir} +./ci/build_wheel.sh ${package_dir} RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 2c609486a81..2d9297e4833 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -13,7 +13,7 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist export PIP_FIND_LINKS="/tmp/libcudf_dist" -./ci/build_wheel.sh cudf ${package_dir} +./ci/build_wheel.sh ${package_dir} cd ${package_dir} mkdir -p final_dist From 98f3eb77432943e058ed9340f0bffe163f1abe74 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 24 Jun 2024 16:42:35 -0500 Subject: [PATCH 17/84] remove hack dependency on Vyas' rapids-cmake branch --- python/libcudf/CMakeLists.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/python/libcudf/CMakeLists.txt b/python/libcudf/CMakeLists.txt index 78a49b553e9..bf2d6e8fcf0 100644 --- a/python/libcudf/CMakeLists.txt +++ b/python/libcudf/CMakeLists.txt @@ -14,9 +14,6 @@ cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) -set(rapids-cmake-repo "vyasr/rapids-cmake") -set(rapids-cmake-branch "fix/nvcomp_lib_dir") - include(../../rapids_config.cmake) project( From 9fbab2eefdaf1b2ebf0157a62c8b82534d73b2c1 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 24 Jun 2024 18:30:13 -0500 Subject: [PATCH 18/84] add runtime pyarrow dep to libcudf --- dependencies.yaml | 7 +++++++ python/libcudf/pyproject.toml | 3 +++ 2 files changed, 10 insertions(+) diff --git a/dependencies.yaml b/dependencies.yaml index d45df84bf82..4f1730d10b8 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -115,6 +115,13 @@ files: - build_cpp - build_python_common - build_python_cudf + py_run_libcudf: + output: pyproject + pyproject_dir: python/libcudf + extras: + table: project + includes: + - pyarrow_run py_run_cudf: output: pyproject pyproject_dir: python/cudf diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index 56439ac06bc..a548872aa88 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -37,6 +37,9 @@ classifiers = [ "Programming Language :: C++", "Environment :: GPU :: NVIDIA CUDA", ] +dependencies = [ + "pyarrow>=16.1.0,<16.2.0a0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls] Homepage = "https://github.com/rapidsai/cudf" From 1d2503dd2359bc234665a364b26e2a2b2eefe259 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 25 Jun 2024 13:00:47 -0500 Subject: [PATCH 19/84] add dependencies-file to libcudf wheel pyproject RBB section --- python/libcudf/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index a548872aa88..a94aa2211fd 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -63,6 +63,7 @@ input = "libcudf/VERSION" regex = "(?P.*)" [tool.rapids-build-backend] +dependencies-file = "../../dependencies.yaml" requires = [ "cmake>=3.26.4", "cython>=3.0.3", From 468301f8dec9d75b36e2f9fd0051d237cea65b5f Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 25 Jun 2024 14:21:41 -0500 Subject: [PATCH 20/84] sort dependencies.yaml entries in build order --- dependencies.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 4f1730d10b8..a5cde514136 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -93,28 +93,28 @@ files: table: build-system includes: - rapids_build_skbuild - py_build_cudf: + py_build_libcudf: output: pyproject - pyproject_dir: python/cudf + pyproject_dir: python/libcudf extras: table: tool.rapids-build-backend key: requires includes: - build_base + - build_cpp - build_python_common - build_python_cudf - - run_libcudf - py_build_libcudf: + py_build_cudf: output: pyproject - pyproject_dir: python/libcudf + pyproject_dir: python/cudf extras: table: tool.rapids-build-backend key: requires includes: - build_base - - build_cpp - build_python_common - build_python_cudf + - run_libcudf py_run_libcudf: output: pyproject pyproject_dir: python/libcudf From 040e5cf23b33ade2828be29abd89c01f68aa6af3 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 25 Jun 2024 14:35:58 -0500 Subject: [PATCH 21/84] add build-backend to RBB tools section --- python/libcudf/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index a94aa2211fd..4b6503dc2eb 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -63,6 +63,7 @@ input = "libcudf/VERSION" regex = "(?P.*)" [tool.rapids-build-backend] +build-backend = "scikit_build_core.build" dependencies-file = "../../dependencies.yaml" requires = [ "cmake>=3.26.4", From 19c6e6674cb14b0576fa2f4650bb8df66fb1efbb Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 25 Jun 2024 14:47:07 -0500 Subject: [PATCH 22/84] change build-backend to RBB in libcudf build-system --- python/libcudf/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index 4b6503dc2eb..316bd28c1d5 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -13,7 +13,7 @@ # limitations under the License. [build-system] -build-backend = "scikit_build_core.build" +build-backend = "rapids_build_backend.build" requires = [ "rapids-build-backend>=0.3.0,<0.4.0.dev0", "scikit-build-core[pyproject]>=0.7.0", From e94816d035b3b3dec09d85d4d14bd14ffe8917d3 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 25 Jun 2024 15:45:12 -0500 Subject: [PATCH 23/84] update libarrow auditwheel exclusions --- ci/build_wheel_cpp.sh | 2 +- ci/build_wheel_python.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 0f8dd2208f9..9694c3f6144 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -10,6 +10,6 @@ package_dir="python/libcudf" RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" mkdir -p ${package_dir}/final_dist -python -m auditwheel repair --exclude libarrow.so.1400 -w ${package_dir}/final_dist ${package_dir}/dist/* +python -m auditwheel repair --exclude libarrow.so.1601 -w ${package_dir}/final_dist ${package_dir}/dist/* RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp ${package_dir}/final_dist diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 2d9297e4833..83dece51f18 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -17,6 +17,6 @@ export PIP_FIND_LINKS="/tmp/libcudf_dist" cd ${package_dir} mkdir -p final_dist -python -m auditwheel repair --exclude libcudf.so --exclude libarrow.so.1400 -w final_dist dist/* +python -m auditwheel repair --exclude libcudf.so --exclude libarrow.so.1601 -w final_dist dist/* RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist From b4d3a62893c98572a4798ba89a3f2e435a7f401d Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 25 Jun 2024 17:10:00 -0500 Subject: [PATCH 24/84] add libcudf -cuXY logic for runtime dep --- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 + conda/environments/all_cuda-122_arch-x86_64.yaml | 1 + dependencies.yaml | 16 ++++++++++++---- python/cudf/pyproject.toml | 4 ++-- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 946e2d1cd32..89cd58c61ca 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -40,6 +40,7 @@ dependencies: - libarrow-acero==16.1.0.* - libarrow-dataset==16.1.0.* - libarrow==16.1.0.* +- libcudf==24.8.* - libcufile-dev=1.4.0.31 - libcufile=1.4.0.31 - libcurand-dev=10.3.0.86 diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml index f069616ddbe..19d574a1a0b 100644 --- a/conda/environments/all_cuda-122_arch-x86_64.yaml +++ b/conda/environments/all_cuda-122_arch-x86_64.yaml @@ -41,6 +41,7 @@ dependencies: - libarrow-acero==16.1.0.* - libarrow-dataset==16.1.0.* - libarrow==16.1.0.* +- libcudf==24.8.* - libcufile-dev - libcurand-dev - libkvikio==24.8.* diff --git a/dependencies.yaml b/dependencies.yaml index a5cde514136..59db5f226e7 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -406,11 +406,19 @@ dependencies: - pyarrow>=16.1.0,<16.2.0a0 run_libcudf: common: - # TODO: Currently this is a hack for devcontainers. Need to figure out the best solution. - #- output_types: [requirements, pyproject] - - output_types: [pyproject] + - output_types: [conda] packages: - - libcudf==24.6.* + - &libcudf_conda libcudf==24.8.* + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: {cuda: "12.*"} + packages: + - libcudf-cu12==24.8.*,>=0.0.0a0 + - matrix: {cuda: "11.*"} + packages: + - libcudf-cu11==24.8.*,>=0.0.0a0 + - {matrix: null, packages: [*libcudf_conda] } cuda_version: specific: - output_types: conda diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 7e776e784ca..44915e6f8ff 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -23,7 +23,7 @@ dependencies = [ "cuda-python>=11.7.1,<12.0a0", "cupy-cuda11x>=12.0.0", "fsspec>=0.6.0", - "libcudf==24.6.*", + "libcudf==24.8.*", "numba>=0.57", "numpy>=1.23,<2.0a0", "nvtx>=0.2.1", @@ -124,7 +124,7 @@ dependencies-file = "../../dependencies.yaml" requires = [ "cmake>=3.26.4", "cython>=3.0.3", - "libcudf==24.6.*", + "libcudf==24.8.*", "ninja", "numpy==1.23.*", "pyarrow==16.1.0.*", From 75043e5d35b20a0fe45f8388fb2ca0ba405171b3 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 25 Jun 2024 18:01:10 -0500 Subject: [PATCH 25/84] add dlpack target include directory --- python/cudf/cudf/_lib/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/python/cudf/cudf/_lib/CMakeLists.txt b/python/cudf/cudf/_lib/CMakeLists.txt index 5a067e84f56..52920cae125 100644 --- a/python/cudf/cudf/_lib/CMakeLists.txt +++ b/python/cudf/cudf/_lib/CMakeLists.txt @@ -64,6 +64,7 @@ rapids_cython_create_modules( ) target_link_libraries(strings_udf PUBLIC cudf_strings_udf) +target_include_directories(interop PUBLIC "$") set(targets_using_arrow_headers interop avro csv orc json parquet) link_to_pyarrow_headers("${targets_using_arrow_headers}") From 470ee3d01c8cfc404b998073e6ec7571817e87fb Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 25 Jun 2024 19:14:07 -0500 Subject: [PATCH 26/84] remove -cuXY from devcontainers build --- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-122_arch-x86_64.yaml | 2 +- dependencies.yaml | 6 +++--- python/cudf/pyproject.toml | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 89cd58c61ca..aa0bdd220ac 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -40,7 +40,7 @@ dependencies: - libarrow-acero==16.1.0.* - libarrow-dataset==16.1.0.* - libarrow==16.1.0.* -- libcudf==24.8.* +- libcudf==24.8.*,>=0.0.0a0 - libcufile-dev=1.4.0.31 - libcufile=1.4.0.31 - libcurand-dev=10.3.0.86 diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml index 19d574a1a0b..18e6cbe033f 100644 --- a/conda/environments/all_cuda-122_arch-x86_64.yaml +++ b/conda/environments/all_cuda-122_arch-x86_64.yaml @@ -41,7 +41,7 @@ dependencies: - libarrow-acero==16.1.0.* - libarrow-dataset==16.1.0.* - libarrow==16.1.0.* -- libcudf==24.8.* +- libcudf==24.8.*,>=0.0.0a0 - libcufile-dev - libcurand-dev - libkvikio==24.8.* diff --git a/dependencies.yaml b/dependencies.yaml index 59db5f226e7..c856e8f60c8 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -406,11 +406,11 @@ dependencies: - pyarrow>=16.1.0,<16.2.0a0 run_libcudf: common: - - output_types: [conda] + - output_types: [conda, requirements] packages: - - &libcudf_conda libcudf==24.8.* + - &libcudf_conda libcudf==24.8.*,>=0.0.0a0 specific: - - output_types: [requirements, pyproject] + - output_types: [pyproject] matrices: - matrix: {cuda: "12.*"} packages: diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 44915e6f8ff..8e6825cc6c0 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -23,7 +23,7 @@ dependencies = [ "cuda-python>=11.7.1,<12.0a0", "cupy-cuda11x>=12.0.0", "fsspec>=0.6.0", - "libcudf==24.8.*", + "libcudf==24.8.*,>=0.0.0a0", "numba>=0.57", "numpy>=1.23,<2.0a0", "nvtx>=0.2.1", @@ -124,7 +124,7 @@ dependencies-file = "../../dependencies.yaml" requires = [ "cmake>=3.26.4", "cython>=3.0.3", - "libcudf==24.8.*", + "libcudf==24.8.*,>=0.0.0a0", "ninja", "numpy==1.23.*", "pyarrow==16.1.0.*", From 2a44eb243ef99b9a35f97f98c49cd4a3a425ddb5 Mon Sep 17 00:00:00 2001 From: Mike Sarahan Date: Wed, 26 Jun 2024 11:10:19 -0500 Subject: [PATCH 27/84] Update .github/workflows/pr.yaml Co-authored-by: Kyle Edwards --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index bcd1977c685..9f0d239b99d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -130,7 +130,7 @@ jobs: wheel-build-python: needs: [checks, wheel-build-cpp] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 with: build_type: pull-request script: "ci/build_wheel_python.sh" From 1e759b6eb32b854bb392f781e5c37455cd359775 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 26 Jun 2024 11:11:06 -0500 Subject: [PATCH 28/84] copy cudf _version.py into libcudf wheel; fix naming convention in deps.yaml --- dependencies.yaml | 8 ++++---- python/libcudf/libcudf/_version.py | 16 +++++++++++++--- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index c856e8f60c8..de75aa81b08 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -78,7 +78,7 @@ files: - libarrow_run - py_version # This is the shared library, bundled as a wheel. It is meant to be consumed by the wrapper. - py_rapids_build_libcudf: + py_build_libcudf: output: pyproject pyproject_dir: python/libcudf extras: @@ -86,14 +86,14 @@ files: includes: - rapids_build_skbuild # This is the wrapper that gets used in Python, not the shared library wheel - py_rapids_build_cudf: + py_build_cudf: output: pyproject pyproject_dir: python/cudf extras: table: build-system includes: - rapids_build_skbuild - py_build_libcudf: + py_rapids_build_libcudf: output: pyproject pyproject_dir: python/libcudf extras: @@ -104,7 +104,7 @@ files: - build_cpp - build_python_common - build_python_cudf - py_build_cudf: + py_rapids_build_cudf: output: pyproject pyproject_dir: python/cudf extras: diff --git a/python/libcudf/libcudf/_version.py b/python/libcudf/libcudf/_version.py index da6b96c163e..7dd732b4905 100644 --- a/python/libcudf/libcudf/_version.py +++ b/python/libcudf/libcudf/_version.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,9 +15,19 @@ import importlib.resources __version__ = ( - importlib.resources.files("libcudf") + importlib.resources.files(__package__) .joinpath("VERSION") .read_text() .strip() ) -__git_commit__ = "" +try: + __git_commit__ = ( + importlib.resources.files(__package__) + .joinpath("GIT_COMMIT") + .read_text() + .strip() + ) +except FileNotFoundError: + __git_commit__ = "" + +__all__ = ["__git_commit__", "__version__"] From 0885f3d0db223e795da217d1288795bacccaa346 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 26 Jun 2024 12:33:37 -0500 Subject: [PATCH 29/84] remove libcudf runtime req for devcontainer (requirements.txt) --- dependencies.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies.yaml b/dependencies.yaml index de75aa81b08..9b34f216056 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -406,7 +406,7 @@ dependencies: - pyarrow>=16.1.0,<16.2.0a0 run_libcudf: common: - - output_types: [conda, requirements] + - output_types: [conda] packages: - &libcudf_conda libcudf==24.8.*,>=0.0.0a0 specific: From 6b856ac86b4013646e84ba5669bcb95908e4c88b Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 26 Jun 2024 12:37:50 -0500 Subject: [PATCH 30/84] add comment about why spdlog is header-only --- cpp/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index ab3489316be..90a80f8d815 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -690,6 +690,8 @@ set_source_files_properties( # libraries that also do the same thing will have their own copies, and they will all be weak # symbols so which one is used is undefined. Marking these symbols as hidden makes them local and # ensures that internal calls to spdlog functions correctly use the local copy. +# Note that it is unusual to use spdlog as a header-only library, but we do so in order to keep +# RMM as a header-only library. set_source_files_properties( src/utilities/logger.cpp PROPERTIES COMPILE_OPTIONS "-fvisibility=hidden;-fvisibility-inlines-hidden" From 32e7252b28a8e8323ca9bfbcb7296d306d761b10 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 26 Jun 2024 12:43:31 -0500 Subject: [PATCH 31/84] format comment --- cpp/CMakeLists.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 90a80f8d815..6dec9510535 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -690,8 +690,9 @@ set_source_files_properties( # libraries that also do the same thing will have their own copies, and they will all be weak # symbols so which one is used is undefined. Marking these symbols as hidden makes them local and # ensures that internal calls to spdlog functions correctly use the local copy. -# Note that it is unusual to use spdlog as a header-only library, but we do so in order to keep -# RMM as a header-only library. +# +# Note that it is unusual to use spdlog as a header-only library, but we do so in order to keep RMM +# as a header-only library. set_source_files_properties( src/utilities/logger.cpp PROPERTIES COMPILE_OPTIONS "-fvisibility=hidden;-fvisibility-inlines-hidden" From a5e2a3b782d5f0a963ba6805924b156fda2cbd33 Mon Sep 17 00:00:00 2001 From: Mike Sarahan Date: Thu, 27 Jun 2024 19:01:32 -0500 Subject: [PATCH 32/84] fix wheel build gha reference --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 519739a870d..4d20fc9de7f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -143,7 +143,7 @@ jobs: build_type: pull-request script: ci/test_wheel_cudf.sh test-cudf-polars: - needs: wheel-build-cudf + needs: wheel-build-python secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 with: From a9ef9ff426bd1aaed54fa31dc7393c378add5400 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 28 Jun 2024 11:39:26 -0500 Subject: [PATCH 33/84] fix missing libcudf wheel in polars test script --- ci/test_cudf_polars.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ci/test_cudf_polars.sh b/ci/test_cudf_polars.sh index 669e049ab26..788df95af36 100755 --- a/ci/test_cudf_polars.sh +++ b/ci/test_cudf_polars.sh @@ -18,16 +18,17 @@ else fi RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist +RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist + +# echo to expand wildcard before adding `[extra]` requires for pip +rapids-logger "Install cudf wheel" +python -m pip install --find-links $(pwd)/dist $(echo ./dist/cudf*.whl)[test] RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ mkdir -p "${RAPIDS_TESTS_DIR}" -rapids-logger "Install cudf wheel" -# echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install $(echo ./dist/cudf*.whl)[test] - rapids-logger "Install polars (allow pre-release versions)" python -m pip install 'polars>=1.0.0a0' From 06b67a877593d66a92f612304da1244135577733 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 28 Jun 2024 12:11:51 -0500 Subject: [PATCH 34/84] elaborate assertion in test_rmm_option_on_import --- python/cudf/cudf_pandas_tests/test_cudf_pandas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py index 0d46e2e9311..13fe6476841 100644 --- a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py +++ b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py @@ -1451,7 +1451,7 @@ def test_rmm_option_on_import(env_value): env=env, ) if env_value in {"cuda", "pool", "async", "managed", "managed_pool"}: - assert sp_completed.returncode == 0 + assert sp_completed.returncode == 0, f"subprocess return code for env_value='{env_value}' was non-zero ({sp_completed.returncode})" else: assert sp_completed.returncode == 1 From 6f005d4d9907715b4b41c9e1b5d7add308453a9c Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 28 Jun 2024 12:33:48 -0500 Subject: [PATCH 35/84] formatting --- python/cudf/cudf_pandas_tests/test_cudf_pandas.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py index 13fe6476841..f3b9a580d1d 100644 --- a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py +++ b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py @@ -1451,7 +1451,9 @@ def test_rmm_option_on_import(env_value): env=env, ) if env_value in {"cuda", "pool", "async", "managed", "managed_pool"}: - assert sp_completed.returncode == 0, f"subprocess return code for env_value='{env_value}' was non-zero ({sp_completed.returncode})" + assert ( + sp_completed.returncode == 0 + ), f"subprocess return code for env_value='{env_value}' was non-zero ({sp_completed.returncode})" else: assert sp_completed.returncode == 1 From 943a08fb1218b697bc99c6fdedbafd4b3a664d3f Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 5 Jul 2024 09:18:29 -0700 Subject: [PATCH 36/84] fix conflicts, stricter installs --- .github/workflows/pr.yaml | 2 +- ci/build_wheel_python.sh | 3 ++- ci/cudf_pandas_scripts/pandas-tests/run.sh | 3 ++- ci/cudf_pandas_scripts/run_tests.sh | 3 ++- ci/test_wheel_cudf.sh | 3 ++- ci/test_wheel_cudf_polars.sh | 6 +++--- 6 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e64803256ee..a0b2f326d9c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -144,7 +144,7 @@ jobs: build_type: pull-request script: ci/test_wheel_cudf.sh wheel-build-cudf-polars: - needs: wheel-build-cudf + needs: wheel-build-python secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 with: diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 83dece51f18..9ca481ad2bd 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -12,7 +12,8 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" # Downloads libcudf wheel from this current build, then points pip to it in PIP_FIND_LINKS below RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist -export PIP_FIND_LINKS="/tmp/libcudf_dist" +echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/libcudf_*.whl)" > /tmp/constraints.txt +export PIP_CONSTRAINT="/tmp/constraints.txt" ./ci/build_wheel.sh ${package_dir} cd ${package_dir} diff --git a/ci/cudf_pandas_scripts/pandas-tests/run.sh b/ci/cudf_pandas_scripts/pandas-tests/run.sh index 562c2c12635..9b9434996cc 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/run.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/run.sh @@ -13,7 +13,8 @@ rapids-logger "PR number: ${RAPIDS_REF_NAME:-"unknown"}" RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-cudf-dep RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./local-cudf-dep -python -m pip install --find-links $(pwd)/local-cudf-dep $(ls ./local-cudf-dep/cudf*.whl)[test,pandas-tests] +python -m pip install "$(echo ./local-cudf-dep/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" +python -m pip install --find-links $(pwd)/local-cudf-dep "$(echo ./local-cudf-dep/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test,pandas-tests]" RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index 0212b67298b..23feb8f45ad 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -38,7 +38,8 @@ else RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-cudf-dep RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./local-cudf-dep - python -m pip install --find-links $(pwd)/local-cudf-dep $(ls ./local-cudf-dep/cudf*.whl)[test,cudf-pandas-tests] + python -m pip install "$(echo ./local-cudf-dep/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" + python -m pip install --find-links $(pwd)/local-cudf-dep "$(echo ./local-cudf-dep/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test,pandas-tests]" fi python -m pytest -p cudf.pandas \ diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh index 27db3767e40..3635fb74a3d 100755 --- a/ci/test_wheel_cudf.sh +++ b/ci/test_wheel_cudf.sh @@ -8,7 +8,8 @@ RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-f RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist # echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install --find-links $(pwd)/dist $(echo ./dist/cudf*.whl)[test] +python -m pip install "$(echo ./local-cudf-dep/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" +python -m pip install --find-links $(pwd)/dist "$(echo ./dist/cudf*.whl)[test]" RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index e99a04d5504..e6d1e6e3c0a 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -24,11 +24,11 @@ RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE # echo to expand wildcard before adding `[extra]` requires for pip rapids-logger "Install cudf wheel" -python -m pip install --find-links $(pwd)/dist $(echo ./dist/cudf_polars*.whl)[test] +python -m pip install "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" +python -m pip install "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" rapids-logger "Install cudf_polars" -python -m pip install 'polars>=1.0.0a0' -python -m pip install --no-deps python/cudf_polars +python -m pip install --find-links $(pwd)/dist "$(echo ./dist/cudf_polars*.whl)[test]" RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ From d4e8bea5fbe1798d15fd71ad9ec5115a0ee86da3 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 8 Jul 2024 08:20:24 -0700 Subject: [PATCH 37/84] fix test script --- ci/test_wheel_cudf.sh | 2 +- python/libcudf/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh index 3635fb74a3d..0fbaf330f8c 100755 --- a/ci/test_wheel_cudf.sh +++ b/ci/test_wheel_cudf.sh @@ -8,7 +8,7 @@ RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-f RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist # echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install "$(echo ./local-cudf-dep/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" +python -m pip install "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" python -m pip install --find-links $(pwd)/dist "$(echo ./dist/cudf*.whl)[test]" RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index 316bd28c1d5..4873d2daaae 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -66,7 +66,7 @@ regex = "(?P.*)" build-backend = "scikit_build_core.build" dependencies-file = "../../dependencies.yaml" requires = [ - "cmake>=3.26.4", + "cmake>=3.26.4,!=3.30.0", "cython>=3.0.3", "ninja", "numpy==1.23.*", From 53125d77be192abe08a5b0d52520c4df4acd45b6 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 8 Jul 2024 13:44:15 -0500 Subject: [PATCH 38/84] get more logs for tests --- ci/build_wheel_python.sh | 7 ++++++- python/cudf/CMakeLists.txt | 5 +++-- python/cudf/cudf_pandas_tests/test_cudf_pandas.py | 5 +++++ python/libcudf/pyproject.toml | 2 +- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 9ca481ad2bd..2c1bb198c06 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -9,7 +9,12 @@ package_dir="python/cudf" export SKBUILD_CMAKE_ARGS="-DUSE_LIBARROW_FROM_PYARROW=ON" RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# Downloads libcudf wheel from this current build, then points pip to it in PIP_FIND_LINKS below + +# Downloads libcudf wheel from this current build, +# then ensures 'cudf' wheel builds always use the 'libcudf' just built in the same CI run. +# +# Using env variable PIP_CONSTRAINT is necessary to ensure the constraints +# are used when creating the isolated build environment. RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/libcudf_*.whl)" > /tmp/constraints.txt diff --git a/python/cudf/CMakeLists.txt b/python/cudf/CMakeLists.txt index 6c11c4c0fe0..8267681defa 100644 --- a/python/cudf/CMakeLists.txt +++ b/python/cudf/CMakeLists.txt @@ -47,11 +47,12 @@ endif() # like headers from libcudacxx, but we do need the libraries. However, we want to control where they # are installed to. Since there are multiple subpackages of cudf._lib that require access to # libcudf, we place the library and all its dependent artifacts in the cudf directory as a single -# source of truth and modify the other rpaths appropriately. +# source of truth. include(cmake/Modules/WheelHelpers.cmake) + # TODO: This install is currently overzealous. We should only install the libraries that are # downloaded by CPM during the build, not libraries that were found on the system. However, in -# practice right this would only be a problem is if libcudf was not found but some of the +# practice this would only be a problem if libcudf was not found but some of the # dependencies were, and we have no real use cases where that happens. install_aliased_imported_targets( TARGETS cudf arrow_shared nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp diff --git a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py index fc0a898a174..f3c97707a37 100644 --- a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py +++ b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py @@ -1451,6 +1451,11 @@ def test_rmm_option_on_import(env_value): env=env, ) if env_value in {"cuda", "pool", "async", "managed", "managed_pool"}: + if sp_completed.returncode != 0: + print("--- stdout") + print(sp_completed.stdout) + print("--- stderr") + print(sp_completed.stderr) assert ( sp_completed.returncode == 0 ), f"subprocess return code for env_value='{env_value}' was non-zero ({sp_completed.returncode})" diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index 4873d2daaae..4d4703fcec3 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -22,7 +22,7 @@ requires = [ [project] name = "libcudf" dynamic = ["version"] -description = "cuDF - GPU Dataframe" +description = "cuDF - GPU Dataframe (C++)" readme = { file = "README.md", content-type = "text/markdown" } authors = [ { name = "NVIDIA Corporation" }, From 42c5450a46c8df341a5673068e05b5801d9ea0b5 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 8 Jul 2024 13:57:05 -0500 Subject: [PATCH 39/84] style --- python/cudf/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cudf/CMakeLists.txt b/python/cudf/CMakeLists.txt index 8267681defa..f9fe5c29cb5 100644 --- a/python/cudf/CMakeLists.txt +++ b/python/cudf/CMakeLists.txt @@ -52,8 +52,8 @@ include(cmake/Modules/WheelHelpers.cmake) # TODO: This install is currently overzealous. We should only install the libraries that are # downloaded by CPM during the build, not libraries that were found on the system. However, in -# practice this would only be a problem if libcudf was not found but some of the -# dependencies were, and we have no real use cases where that happens. +# practice this would only be a problem if libcudf was not found but some of the dependencies were, +# and we have no real use cases where that happens. install_aliased_imported_targets( TARGETS cudf arrow_shared nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp DESTINATION ${cython_lib_dir} From 9d05dfc5adddcd314f22549dfcc20c34eeb58e75 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 8 Jul 2024 15:42:43 -0700 Subject: [PATCH 40/84] fix extra --- ci/cudf_pandas_scripts/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index 23feb8f45ad..b452eabc6ec 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -39,7 +39,7 @@ else RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-cudf-dep RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./local-cudf-dep python -m pip install "$(echo ./local-cudf-dep/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" - python -m pip install --find-links $(pwd)/local-cudf-dep "$(echo ./local-cudf-dep/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test,pandas-tests]" + python -m pip install --find-links $(pwd)/local-cudf-dep "$(echo ./local-cudf-dep/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test,cudf-pandas-tests]" fi python -m pytest -p cudf.pandas \ From 020ea50297825e78e81683bbdf63c3be626e5a2e Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 29 Jul 2024 12:40:29 -0500 Subject: [PATCH 41/84] fix update-version.sh, some 24.8 references --- .github/workflows/pr.yaml | 2 +- .github/workflows/pr_issue_status_automation.yml | 6 +++--- ci/release/update-version.sh | 1 + conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-125_arch-x86_64.yaml | 2 +- dependencies.yaml | 6 +++--- python/cudf/pyproject.toml | 4 ++-- python/libcudf/pyproject.toml | 2 +- 8 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index d2665022ea1..1ac61006d9f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -132,7 +132,7 @@ jobs: wheel-build-python: needs: [checks, wheel-build-cpp] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: pull-request script: "ci/build_wheel_python.sh" diff --git a/.github/workflows/pr_issue_status_automation.yml b/.github/workflows/pr_issue_status_automation.yml index 8ca971dc28d..45e5191eb54 100644 --- a/.github/workflows/pr_issue_status_automation.yml +++ b/.github/workflows/pr_issue_status_automation.yml @@ -23,7 +23,7 @@ on: jobs: get-project-id: - uses: rapidsai/shared-workflows/.github/workflows/project-get-item-id.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/project-get-item-id.yaml@branch-24.10 if: github.event.pull_request.state == 'open' secrets: inherit permissions: @@ -34,7 +34,7 @@ jobs: update-status: # This job sets the PR and its linked issues to "In Progress" status - uses: rapidsai/shared-workflows/.github/workflows/project-get-set-single-select-field.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/project-get-set-single-select-field.yaml@branch-24.10 if: ${{ github.event.pull_request.state == 'open' && needs.get-project-id.outputs.ITEM_PROJECT_ID != '' }} needs: get-project-id with: @@ -50,7 +50,7 @@ jobs: update-sprint: # This job sets the PR and its linked issues to the current "Weekly Sprint" - uses: rapidsai/shared-workflows/.github/workflows/project-get-set-iteration-field.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/project-get-set-iteration-field.yaml@branch-24.10 if: ${{ github.event.pull_request.state == 'open' && needs.get-project-id.outputs.ITEM_PROJECT_ID != '' }} needs: get-project-id with: diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index ad96aff3930..ab9d6e12a7d 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -51,6 +51,7 @@ DEPENDENCIES=( dask-cuda dask-cudf kvikio + libcudf libkvikio librmm rapids-dask-dependency diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 4a6c00bf61a..28b44396fc7 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -39,7 +39,7 @@ dependencies: - libarrow-acero==16.1.0.* - libarrow-dataset==16.1.0.* - libarrow==16.1.0.* -- libcudf==24.8.*,>=0.0.0a0 +- libcudf==24.10.*,>=0.0.0a0 - libcufile-dev=1.4.0.31 - libcufile=1.4.0.31 - libcurand-dev=10.3.0.86 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index eb613950c2a..97396470fad 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -40,7 +40,7 @@ dependencies: - libarrow-acero==16.1.0.* - libarrow-dataset==16.1.0.* - libarrow==16.1.0.* -- libcudf==24.8.*,>=0.0.0a0 +- libcudf==24.10.*,>=0.0.0a0 - libcufile-dev - libcurand-dev - libkvikio==24.10.*,>=0.0.0a0 diff --git a/dependencies.yaml b/dependencies.yaml index 24a017f7dbe..3831a5fa918 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -413,16 +413,16 @@ dependencies: common: - output_types: [conda] packages: - - &libcudf_conda libcudf==24.8.*,>=0.0.0a0 + - &libcudf_conda libcudf==24.10.*,>=0.0.0a0 specific: - output_types: [pyproject] matrices: - matrix: {cuda: "12.*"} packages: - - libcudf-cu12==24.8.*,>=0.0.0a0 + - libcudf-cu12==24.10.*,>=0.0.0a0 - matrix: {cuda: "11.*"} packages: - - libcudf-cu11==24.8.*,>=0.0.0a0 + - libcudf-cu11==24.10.*,>=0.0.0a0 - {matrix: null, packages: [*libcudf_conda] } cuda_version: specific: diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 55074a68121..34bfc7cfc22 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -23,7 +23,7 @@ dependencies = [ "cuda-python>=11.7.1,<12.0a0", "cupy-cuda11x>=12.0.0", "fsspec>=0.6.0", - "libcudf==24.8.*,>=0.0.0a0", + "libcudf==24.10.*,>=0.0.0a0", "numba>=0.57", "numpy>=1.23,<2.0a0", "nvtx>=0.2.1", @@ -125,7 +125,7 @@ matrix-entry = "cuda_suffixed=true" requires = [ "cmake>=3.26.4,!=3.30.0", "cython>=3.0.3", - "libcudf==24.8.*,>=0.0.0a0", + "libcudf==24.10.*,>=0.0.0a0", "ninja", "numpy==1.23.*", "pyarrow==16.1.0.*", diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index 4d4703fcec3..cad3dde52d2 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -71,5 +71,5 @@ requires = [ "ninja", "numpy==1.23.*", "pyarrow==16.1.0.*", - "rmm==24.8.*,>=0.0.0a0", + "rmm==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From 1cb8dcccaca7b4d45c9ec92fad7100aa91ec524a Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 29 Jul 2024 13:01:07 -0500 Subject: [PATCH 42/84] need cuda_suffixed=true --- python/libcudf/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index cad3dde52d2..d70d9f3132a 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -65,6 +65,7 @@ regex = "(?P.*)" [tool.rapids-build-backend] build-backend = "scikit_build_core.build" dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true" requires = [ "cmake>=3.26.4,!=3.30.0", "cython>=3.0.3", From b5223b1e7ec7be808387e6917237a286c8d8abe9 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 8 Aug 2024 09:55:07 -0700 Subject: [PATCH 43/84] update scikit-build-core --- python/libcudf/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index d70d9f3132a..486190c1a10 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -16,7 +16,7 @@ build-backend = "rapids_build_backend.build" requires = [ "rapids-build-backend>=0.3.0,<0.4.0.dev0", - "scikit-build-core[pyproject]>=0.7.0", + "scikit-build-core[pyproject]>=0.10.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project] From 0ccfd0600ec1029fec2e7505a8c431625949d2dc Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 9 Aug 2024 09:23:37 -0700 Subject: [PATCH 44/84] use rmm/librmm wheels from https://github.com/rapidsai/rmm/pull/1644, temporarily skip conda jobs --- .github/workflows/pr.yaml | 180 ++++++++++++++-------------- ci/build_wheel_cpp.sh | 22 ++++ ci/build_wheel_python.sh | 22 ++++ ci/cudf_pandas_scripts/run_tests.sh | 24 ++++ 4 files changed, 158 insertions(+), 90 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 1ac61006d9f..82d80a7aa5a 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -13,16 +13,16 @@ jobs: pr-builder: needs: - checks - - conda-cpp-build - - conda-cpp-checks - - conda-cpp-tests - - conda-python-build - - conda-python-cudf-tests - - conda-python-other-tests - - conda-java-tests + # - conda-cpp-build + # - conda-cpp-checks + # - conda-cpp-tests + # - conda-python-build + # - conda-python-cudf-tests + # - conda-python-other-tests + # - conda-java-tests - static-configure - - conda-notebook-tests - - docs-build + # - conda-notebook-tests + # - docs-build - wheel-build-cpp - wheel-build-python - wheel-tests-python @@ -30,7 +30,7 @@ jobs: - wheel-tests-cudf-polars - wheel-build-dask-cudf - wheel-tests-dask-cudf - - devcontainer + # - devcontainer - unit-tests-cudf-pandas - pandas-tests - pandas-tests-diff @@ -41,56 +41,56 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 with: enable_check_generated_files: false - conda-cpp-build: - needs: checks - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 - with: - build_type: pull-request - conda-cpp-checks: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 - with: - build_type: pull-request - enable_check_symbols: true - conda-cpp-tests: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 - with: - build_type: pull-request - conda-python-build: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 - with: - build_type: pull-request - conda-python-cudf-tests: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 - with: - build_type: pull-request - script: "ci/test_python_cudf.sh" - conda-python-other-tests: - # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 - with: - build_type: pull-request - script: "ci/test_python_other.sh" - conda-java-tests: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/test_java.sh" + # conda-cpp-build: + # needs: checks + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 + # with: + # build_type: pull-request + # conda-cpp-checks: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 + # with: + # build_type: pull-request + # enable_check_symbols: true + # conda-cpp-tests: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 + # with: + # build_type: pull-request + # conda-python-build: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 + # with: + # build_type: pull-request + # conda-python-cudf-tests: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + # with: + # build_type: pull-request + # script: "ci/test_python_cudf.sh" + # conda-python-other-tests: + # # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + # with: + # build_type: pull-request + # script: "ci/test_python_other.sh" + # conda-java-tests: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + # with: + # build_type: pull-request + # node_type: "gpu-v100-latest-1" + # arch: "amd64" + # container_image: "rapidsai/ci-conda:latest" + # run_script: "ci/test_java.sh" static-configure: needs: checks secrets: inherit @@ -101,26 +101,26 @@ jobs: # primary static consumers (Spark) are not in conda anyway. container_image: "rapidsai/ci-wheel:latest" run_script: "ci/configure_cpp_static.sh" - conda-notebook-tests: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/test_notebooks.sh" - docs-build: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/build_docs.sh" + # conda-notebook-tests: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + # with: + # build_type: pull-request + # node_type: "gpu-v100-latest-1" + # arch: "amd64" + # container_image: "rapidsai/ci-conda:latest" + # run_script: "ci/test_notebooks.sh" + # docs-build: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + # with: + # build_type: pull-request + # node_type: "gpu-v100-latest-1" + # arch: "amd64" + # container_image: "rapidsai/ci-conda:latest" + # run_script: "ci/build_docs.sh" wheel-build-cpp: needs: checks secrets: inherit @@ -181,16 +181,16 @@ jobs: matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: pull-request script: ci/test_wheel_dask_cudf.sh - devcontainer: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.10 - with: - arch: '["amd64"]' - cuda: '["12.5"]' - build_command: | - sccache -z; - build-all -DBUILD_BENCHMARKS=ON --verbose; - sccache -s; + # devcontainer: + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.10 + # with: + # arch: '["amd64"]' + # cuda: '["12.5"]' + # build_command: | + # sccache -z; + # build-all -DBUILD_BENCHMARKS=ON --verbose; + # sccache -s; unit-tests-cudf-pandas: needs: wheel-build-python secrets: inherit diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 9694c3f6144..3b2269dd0cc 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -5,6 +5,28 @@ set -euo pipefail package_dir="python/libcudf" +# --- start of section to remove ---# +# TODO: remove this before merging +# use librmm and rmm from +RAPIDS_REPOSITORY=rmm \ +RAPIDS_BUILD_TYPE=pull-requst \ +RAPIDS_REF_NAME=1644 \ +RAPIDS_SHA=0701559 \ +RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" \ + rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist + +RAPIDS_REPOSITORY=rmm \ +RAPIDS_BUILD_TYPE=pull-requst \ +RAPIDS_REF_NAME=1644 \ +RAPIDS_SHA=0701559 \ +RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ + rapids-download-wheels-from-s3 python /tmp/libcudf_dist + +echo "librmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/librmm_*.whl)" >> /tmp/constraints.txt +echo "rmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/librmm_*.whl)" >> /tmp/constraints.txt +export PIP_CONSTRAINT=/tmp/constraints.txt +# --- end of section to remove ---# + ./ci/build_wheel.sh ${package_dir} RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 2c1bb198c06..222a56fc375 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -18,6 +18,28 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/libcudf_*.whl)" > /tmp/constraints.txt + +# --- start of section to remove ---# +# TODO: remove this before merging +# use librmm and rmm from +RAPIDS_REPOSITORY=rmm \ +RAPIDS_BUILD_TYPE=pull-requst \ +RAPIDS_REF_NAME=1644 \ +RAPIDS_SHA=0701559 \ +RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" \ + rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist + +RAPIDS_REPOSITORY=rmm \ +RAPIDS_BUILD_TYPE=pull-requst \ +RAPIDS_REF_NAME=1644 \ +RAPIDS_SHA=0701559 \ +RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ + rapids-download-wheels-from-s3 python /tmp/libcudf_dist + +echo "librmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/librmm_*.whl)" >> /tmp/constraints.txt +echo "rmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/librmm_*.whl)" >> /tmp/constraints.txt +# --- end of section to remove ---# + export PIP_CONSTRAINT="/tmp/constraints.txt" ./ci/build_wheel.sh ${package_dir} diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index b452eabc6ec..bddd30316d2 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -38,6 +38,30 @@ else RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-cudf-dep RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./local-cudf-dep + + # --- start of section to remove ---# + # TODO: remove this before merging + # use librmm and rmm from + RAPIDS_REPOSITORY=rmm \ + RAPIDS_BUILD_TYPE=pull-requst \ + RAPIDS_REF_NAME=1644 \ + RAPIDS_SHA=0701559 \ + RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" \ + rapids-download-wheels-from-s3 cpp /tmp/local-rmm-dep + + RAPIDS_REPOSITORY=rmm \ + RAPIDS_BUILD_TYPE=pull-requst \ + RAPIDS_REF_NAME=1644 \ + RAPIDS_SHA=0701559 \ + RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ + rapids-download-wheels-from-s3 python /tmp/local-rmm-dep + + echo "librmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/local-rmm-dep/librmm_*.whl)" >> /tmp/constraints.txt + echo "rmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/local-rmm-dep/librmm_*.whl)" >> /tmp/constraints.txt + + export PIP_CONSTRAINT=/tmp/constraints.txt + # --- end of section to remove ---# + python -m pip install "$(echo ./local-cudf-dep/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" python -m pip install --find-links $(pwd)/local-cudf-dep "$(echo ./local-cudf-dep/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test,cudf-pandas-tests]" fi From 0d0e291676461b5535eff17a8f36b32570d85549 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 9 Aug 2024 09:31:15 -0700 Subject: [PATCH 45/84] fix typos --- ci/build_wheel_cpp.sh | 4 ++-- ci/build_wheel_python.sh | 4 ++-- ci/cudf_pandas_scripts/run_tests.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 3b2269dd0cc..3b1feeee318 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -9,14 +9,14 @@ package_dir="python/libcudf" # TODO: remove this before merging # use librmm and rmm from RAPIDS_REPOSITORY=rmm \ -RAPIDS_BUILD_TYPE=pull-requst \ +RAPIDS_BUILD_TYPE=pull-request \ RAPIDS_REF_NAME=1644 \ RAPIDS_SHA=0701559 \ RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" \ rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist RAPIDS_REPOSITORY=rmm \ -RAPIDS_BUILD_TYPE=pull-requst \ +RAPIDS_BUILD_TYPE=pull-request \ RAPIDS_REF_NAME=1644 \ RAPIDS_SHA=0701559 \ RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 222a56fc375..f955ebfca4a 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -23,14 +23,14 @@ echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/libcudf # TODO: remove this before merging # use librmm and rmm from RAPIDS_REPOSITORY=rmm \ -RAPIDS_BUILD_TYPE=pull-requst \ +RAPIDS_BUILD_TYPE=pull-request \ RAPIDS_REF_NAME=1644 \ RAPIDS_SHA=0701559 \ RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" \ rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist RAPIDS_REPOSITORY=rmm \ -RAPIDS_BUILD_TYPE=pull-requst \ +RAPIDS_BUILD_TYPE=pull-request \ RAPIDS_REF_NAME=1644 \ RAPIDS_SHA=0701559 \ RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index bddd30316d2..6a4e16d0241 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -43,14 +43,14 @@ else # TODO: remove this before merging # use librmm and rmm from RAPIDS_REPOSITORY=rmm \ - RAPIDS_BUILD_TYPE=pull-requst \ + RAPIDS_BUILD_TYPE=pull-request \ RAPIDS_REF_NAME=1644 \ RAPIDS_SHA=0701559 \ RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" \ rapids-download-wheels-from-s3 cpp /tmp/local-rmm-dep RAPIDS_REPOSITORY=rmm \ - RAPIDS_BUILD_TYPE=pull-requst \ + RAPIDS_BUILD_TYPE=pull-request \ RAPIDS_REF_NAME=1644 \ RAPIDS_SHA=0701559 \ RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ From 8285d1d28106db340bcc5923af5151b4b91e6424 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 9 Aug 2024 09:46:42 -0700 Subject: [PATCH 46/84] set RAPIDS_PY_CUDA_SUFFIX --- ci/build_wheel_cpp.sh | 5 ++++- ci/build_wheel_python.sh | 2 +- ci/cudf_pandas_scripts/run_tests.sh | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 3b1feeee318..45837977703 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -7,7 +7,10 @@ package_dir="python/libcudf" # --- start of section to remove ---# # TODO: remove this before merging -# use librmm and rmm from + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +# use librmm and rmm from https://github.com/rapidsai/rmm/pull/1644 RAPIDS_REPOSITORY=rmm \ RAPIDS_BUILD_TYPE=pull-request \ RAPIDS_REF_NAME=1644 \ diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index f955ebfca4a..882faf4015f 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -21,7 +21,7 @@ echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/libcudf # --- start of section to remove ---# # TODO: remove this before merging -# use librmm and rmm from +# use librmm and rmm from https://github.com/rapidsai/rmm/pull/1644 RAPIDS_REPOSITORY=rmm \ RAPIDS_BUILD_TYPE=pull-request \ RAPIDS_REF_NAME=1644 \ diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index 6a4e16d0241..7fb495016ef 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -41,7 +41,7 @@ else # --- start of section to remove ---# # TODO: remove this before merging - # use librmm and rmm from + # use librmm and rmm from https://github.com/rapidsai/rmm/pull/1644 RAPIDS_REPOSITORY=rmm \ RAPIDS_BUILD_TYPE=pull-request \ RAPIDS_REF_NAME=1644 \ From 212388a58e752bfc286c64ea7ba5b7162a194aeb Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 9 Aug 2024 12:02:52 -0500 Subject: [PATCH 47/84] artifact name is rmm_ for both cpp and python wheels --- ci/build_wheel_cpp.sh | 4 ++-- ci/build_wheel_python.sh | 4 ++-- ci/cudf_pandas_scripts/run_tests.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 45837977703..2e2163b96eb 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -15,7 +15,7 @@ RAPIDS_REPOSITORY=rmm \ RAPIDS_BUILD_TYPE=pull-request \ RAPIDS_REF_NAME=1644 \ RAPIDS_SHA=0701559 \ -RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" \ +RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist RAPIDS_REPOSITORY=rmm \ @@ -26,7 +26,7 @@ RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ rapids-download-wheels-from-s3 python /tmp/libcudf_dist echo "librmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/librmm_*.whl)" >> /tmp/constraints.txt -echo "rmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/librmm_*.whl)" >> /tmp/constraints.txt +echo "rmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/rmm_*.whl)" >> /tmp/constraints.txt export PIP_CONSTRAINT=/tmp/constraints.txt # --- end of section to remove ---# diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 882faf4015f..1d5fea1c6f3 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -26,7 +26,7 @@ RAPIDS_REPOSITORY=rmm \ RAPIDS_BUILD_TYPE=pull-request \ RAPIDS_REF_NAME=1644 \ RAPIDS_SHA=0701559 \ -RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" \ +RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist RAPIDS_REPOSITORY=rmm \ @@ -37,7 +37,7 @@ RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ rapids-download-wheels-from-s3 python /tmp/libcudf_dist echo "librmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/librmm_*.whl)" >> /tmp/constraints.txt -echo "rmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/librmm_*.whl)" >> /tmp/constraints.txt +echo "rmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/rmm_*.whl)" >> /tmp/constraints.txt # --- end of section to remove ---# export PIP_CONSTRAINT="/tmp/constraints.txt" diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index 7fb495016ef..c4f29437f41 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -46,7 +46,7 @@ else RAPIDS_BUILD_TYPE=pull-request \ RAPIDS_REF_NAME=1644 \ RAPIDS_SHA=0701559 \ - RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" \ + RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ rapids-download-wheels-from-s3 cpp /tmp/local-rmm-dep RAPIDS_REPOSITORY=rmm \ @@ -57,7 +57,7 @@ else rapids-download-wheels-from-s3 python /tmp/local-rmm-dep echo "librmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/local-rmm-dep/librmm_*.whl)" >> /tmp/constraints.txt - echo "rmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/local-rmm-dep/librmm_*.whl)" >> /tmp/constraints.txt + echo "rmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/local-rmm-dep/rmm_*.whl)" >> /tmp/constraints.txt export PIP_CONSTRAINT=/tmp/constraints.txt # --- end of section to remove ---# From 06474bd7c792471513c75fa5f5b6abb2f2f1166c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 9 Aug 2024 12:41:36 -0500 Subject: [PATCH 48/84] missed a spot where rmm and librmm are installed --- ci/cudf_pandas_scripts/pandas-tests/run.sh | 25 ++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/ci/cudf_pandas_scripts/pandas-tests/run.sh b/ci/cudf_pandas_scripts/pandas-tests/run.sh index e13bb67df0b..f6d57ebc079 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/run.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/run.sh @@ -13,6 +13,31 @@ rapids-logger "PR number: ${RAPIDS_REF_NAME:-"unknown"}" RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-cudf-dep RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./local-cudf-dep + +# --- start of section to remove ---# +# TODO: remove this before merging +# use librmm and rmm from https://github.com/rapidsai/rmm/pull/1644 +RAPIDS_REPOSITORY=rmm \ +RAPIDS_BUILD_TYPE=pull-request \ +RAPIDS_REF_NAME=1644 \ +RAPIDS_SHA=0701559 \ +RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ + rapids-download-wheels-from-s3 cpp /tmp/local-rmm-dep + +RAPIDS_REPOSITORY=rmm \ +RAPIDS_BUILD_TYPE=pull-request \ +RAPIDS_REF_NAME=1644 \ +RAPIDS_SHA=0701559 \ +RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ + rapids-download-wheels-from-s3 python /tmp/local-rmm-dep + +echo "librmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/local-rmm-dep/librmm_*.whl)" >> /tmp/constraints.txt +echo "rmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/local-rmm-dep/rmm_*.whl)" >> /tmp/constraints.txt + +export PIP_CONSTRAINT=/tmp/constraints.txt +# --- end of section to remove ---# + + python -m pip install "$(echo ./local-cudf-dep/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" python -m pip install --find-links $(pwd)/local-cudf-dep "$(echo ./local-cudf-dep/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test,pandas-tests]" From c24210a882b43cd0a8a7c3bb9f37faa61c564787 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 13 Aug 2024 06:30:41 -0700 Subject: [PATCH 49/84] update to latest rmm --- ci/build_wheel_cpp.sh | 4 ++-- ci/build_wheel_python.sh | 4 ++-- ci/cudf_pandas_scripts/pandas-tests/run.sh | 4 ++-- ci/cudf_pandas_scripts/run_tests.sh | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 2e2163b96eb..5539499487e 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -14,14 +14,14 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_REPOSITORY=rmm \ RAPIDS_BUILD_TYPE=pull-request \ RAPIDS_REF_NAME=1644 \ -RAPIDS_SHA=0701559 \ +RAPIDS_SHA=e93f26c \ RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist RAPIDS_REPOSITORY=rmm \ RAPIDS_BUILD_TYPE=pull-request \ RAPIDS_REF_NAME=1644 \ -RAPIDS_SHA=0701559 \ +RAPIDS_SHA=e93f26c \ RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ rapids-download-wheels-from-s3 python /tmp/libcudf_dist diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 1d5fea1c6f3..1317f5ce25c 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -25,14 +25,14 @@ echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/libcudf RAPIDS_REPOSITORY=rmm \ RAPIDS_BUILD_TYPE=pull-request \ RAPIDS_REF_NAME=1644 \ -RAPIDS_SHA=0701559 \ +RAPIDS_SHA=e93f26c \ RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist RAPIDS_REPOSITORY=rmm \ RAPIDS_BUILD_TYPE=pull-request \ RAPIDS_REF_NAME=1644 \ -RAPIDS_SHA=0701559 \ +RAPIDS_SHA=e93f26c \ RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ rapids-download-wheels-from-s3 python /tmp/libcudf_dist diff --git a/ci/cudf_pandas_scripts/pandas-tests/run.sh b/ci/cudf_pandas_scripts/pandas-tests/run.sh index f6d57ebc079..55728425f95 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/run.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/run.sh @@ -20,14 +20,14 @@ RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from RAPIDS_REPOSITORY=rmm \ RAPIDS_BUILD_TYPE=pull-request \ RAPIDS_REF_NAME=1644 \ -RAPIDS_SHA=0701559 \ +RAPIDS_SHA=e93f26c \ RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ rapids-download-wheels-from-s3 cpp /tmp/local-rmm-dep RAPIDS_REPOSITORY=rmm \ RAPIDS_BUILD_TYPE=pull-request \ RAPIDS_REF_NAME=1644 \ -RAPIDS_SHA=0701559 \ +RAPIDS_SHA=e93f26c \ RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ rapids-download-wheels-from-s3 python /tmp/local-rmm-dep diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index c4f29437f41..af3270f3bd5 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -45,14 +45,14 @@ else RAPIDS_REPOSITORY=rmm \ RAPIDS_BUILD_TYPE=pull-request \ RAPIDS_REF_NAME=1644 \ - RAPIDS_SHA=0701559 \ + RAPIDS_SHA=e93f26c \ RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ rapids-download-wheels-from-s3 cpp /tmp/local-rmm-dep RAPIDS_REPOSITORY=rmm \ RAPIDS_BUILD_TYPE=pull-request \ RAPIDS_REF_NAME=1644 \ - RAPIDS_SHA=0701559 \ + RAPIDS_SHA=e93f26c \ RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ rapids-download-wheels-from-s3 python /tmp/local-rmm-dep From 9c9536be6eafd04d2f0ec9b2c73f4d60eaaa2205 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 13 Aug 2024 13:49:22 -0700 Subject: [PATCH 50/84] uncomment all CI, logger-specific visibility control should no longer be necessary --- .github/workflows/pr.yaml | 180 +++++++++++++++++++------------------- cpp/CMakeLists.txt | 12 --- 2 files changed, 90 insertions(+), 102 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index fcbf3c48873..9802b238931 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -13,16 +13,16 @@ jobs: pr-builder: needs: - checks - # - conda-cpp-build - # - conda-cpp-checks - # - conda-cpp-tests - # - conda-python-build - # - conda-python-cudf-tests - # - conda-python-other-tests - # - conda-java-tests + - conda-cpp-build + - conda-cpp-checks + - conda-cpp-tests + - conda-python-build + - conda-python-cudf-tests + - conda-python-other-tests + - conda-java-tests - static-configure - # - conda-notebook-tests - # - docs-build + - conda-notebook-tests + - docs-build - wheel-build-cpp - wheel-build-python - wheel-tests-python @@ -30,7 +30,7 @@ jobs: - wheel-tests-cudf-polars - wheel-build-dask-cudf - wheel-tests-dask-cudf - # - devcontainer + - devcontainer - unit-tests-cudf-pandas - pandas-tests - pandas-tests-diff @@ -41,56 +41,56 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 with: enable_check_generated_files: false - # conda-cpp-build: - # needs: checks - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 - # with: - # build_type: pull-request - # conda-cpp-checks: - # needs: conda-cpp-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 - # with: - # build_type: pull-request - # enable_check_symbols: true - # conda-cpp-tests: - # needs: conda-cpp-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 - # with: - # build_type: pull-request - # conda-python-build: - # needs: conda-cpp-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 - # with: - # build_type: pull-request - # conda-python-cudf-tests: - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 - # with: - # build_type: pull-request - # script: "ci/test_python_cudf.sh" - # conda-python-other-tests: - # # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 - # with: - # build_type: pull-request - # script: "ci/test_python_other.sh" - # conda-java-tests: - # needs: conda-cpp-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - # with: - # build_type: pull-request - # node_type: "gpu-v100-latest-1" - # arch: "amd64" - # container_image: "rapidsai/ci-conda:latest" - # run_script: "ci/test_java.sh" + conda-cpp-build: + needs: checks + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 + with: + build_type: pull-request + conda-cpp-checks: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 + with: + build_type: pull-request + enable_check_symbols: true + conda-cpp-tests: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 + with: + build_type: pull-request + conda-python-build: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 + with: + build_type: pull-request + conda-python-cudf-tests: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + with: + build_type: pull-request + script: "ci/test_python_cudf.sh" + conda-python-other-tests: + # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + with: + build_type: pull-request + script: "ci/test_python_other.sh" + conda-java-tests: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: "ci/test_java.sh" static-configure: needs: checks secrets: inherit @@ -101,26 +101,26 @@ jobs: # primary static consumers (Spark) are not in conda anyway. container_image: "rapidsai/ci-wheel:latest" run_script: "ci/configure_cpp_static.sh" - # conda-notebook-tests: - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - # with: - # build_type: pull-request - # node_type: "gpu-v100-latest-1" - # arch: "amd64" - # container_image: "rapidsai/ci-conda:latest" - # run_script: "ci/test_notebooks.sh" - # docs-build: - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - # with: - # build_type: pull-request - # node_type: "gpu-v100-latest-1" - # arch: "amd64" - # container_image: "rapidsai/ci-conda:latest" - # run_script: "ci/build_docs.sh" + conda-notebook-tests: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: "ci/test_notebooks.sh" + docs-build: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: "ci/build_docs.sh" wheel-build-cpp: needs: checks secrets: inherit @@ -181,16 +181,16 @@ jobs: matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: pull-request script: ci/test_wheel_dask_cudf.sh - # devcontainer: - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.10 - # with: - # arch: '["amd64"]' - # cuda: '["12.5"]' - # build_command: | - # sccache -z; - # build-all -DBUILD_BENCHMARKS=ON --verbose; - # sccache -s; + devcontainer: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.10 + with: + arch: '["amd64"]' + cuda: '["12.5"]' + build_command: | + sccache -z; + build-all -DBUILD_BENCHMARKS=ON --verbose; + sccache -s; unit-tests-cudf-pandas: needs: wheel-build-python secrets: inherit diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index c5fd1283356..eeafc411874 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -695,18 +695,6 @@ set_source_files_properties( src/transform/transform.cpp PROPERTIES COMPILE_DEFINITIONS "_FILE_OFFSET_BITS=64" ) -# We need to hide symbols in the logger TU by default because those symbols include spdlog symbols -# that are compiled in because we use it as a header-only library. This is problematic because other -# libraries that also do the same thing will have their own copies, and they will all be weak -# symbols so which one is used is undefined. Marking these symbols as hidden makes them local and -# ensures that internal calls to spdlog functions correctly use the local copy. -# -# Note that it is unusual to use spdlog as a header-only library, but we do so in order to keep RMM -# as a header-only library. -set_source_files_properties( - src/utilities/logger.cpp PROPERTIES COMPILE_OPTIONS - "-fvisibility=hidden;-fvisibility-inlines-hidden" -) set_property( SOURCE src/io/parquet/writer_impl.cu From d97e19ff286aa6e1c664ac22b7de99f43e66d63c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 14 Aug 2024 08:10:08 -0700 Subject: [PATCH 51/84] new pattern for scikit-build-core config, remove some debugging-only stuff --- dependencies.yaml | 8 ++++++-- python/cudf/cudf_pandas_tests/test_cudf_pandas.py | 9 +-------- python/libcudf/pyproject.toml | 3 ++- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 484eaf5211c..ccfa0931020 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -418,10 +418,14 @@ dependencies: specific: - output_types: [pyproject] matrices: - - matrix: {cuda: "12.*"} + - matrix: + cuda: "12.*" + cuda_suffixed: "true" packages: - libcudf-cu12==24.10.*,>=0.0.0a0 - - matrix: {cuda: "11.*"} + - matrix: + cuda: "11.*" + cuda_suffixed: "true" packages: - libcudf-cu11==24.10.*,>=0.0.0a0 - {matrix: null, packages: [*libcudf_conda] } diff --git a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py index 88101e0f6eb..6292022d8e4 100644 --- a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py +++ b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py @@ -1458,14 +1458,7 @@ def test_rmm_option_on_import(env_value): env=env, ) if env_value in {"cuda", "pool", "async", "managed", "managed_pool"}: - if sp_completed.returncode != 0: - print("--- stdout") - print(sp_completed.stdout) - print("--- stderr") - print(sp_completed.stderr) - assert ( - sp_completed.returncode == 0 - ), f"subprocess return code for env_value='{env_value}' was non-zero ({sp_completed.returncode})" + assert sp_completed.returncode == 0 else: assert sp_completed.returncode == 1 diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index 486190c1a10..a02fc434637 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -50,7 +50,8 @@ libcudf = "libcudf" [tool.scikit-build] build-dir = "build/{wheel_tag}" cmake.build-type = "Release" -cmake.minimum-version = "3.26.4" +cmake.version = "CMakeLists.txt" +minimum-version = "build-system.requires" ninja.make-fallback = true sdist.reproducible = true wheel.packages = ["libcudf"] From a4ffdc8f2ab29e3e028b36bd7b842f7d55f475fe Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 15 Aug 2024 07:25:47 -0700 Subject: [PATCH 52/84] try to use regular rmm nightlies, remove other debugging stuff, combine pip installs --- ci/build_wheel_cpp.sh | 25 ------------------- ci/build_wheel_python.sh | 21 ---------------- ci/cudf_pandas_scripts/pandas-tests/run.sh | 29 +++------------------- ci/cudf_pandas_scripts/run_tests.sh | 28 +++------------------ ci/test_wheel_cudf.sh | 5 ++-- ci/test_wheel_cudf_polars.sh | 11 ++++---- ci/test_wheel_dask_cudf.sh | 6 +++-- dependencies.yaml | 4 +-- 8 files changed, 20 insertions(+), 109 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 5539499487e..9694c3f6144 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -5,31 +5,6 @@ set -euo pipefail package_dir="python/libcudf" -# --- start of section to remove ---# -# TODO: remove this before merging - -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" - -# use librmm and rmm from https://github.com/rapidsai/rmm/pull/1644 -RAPIDS_REPOSITORY=rmm \ -RAPIDS_BUILD_TYPE=pull-request \ -RAPIDS_REF_NAME=1644 \ -RAPIDS_SHA=e93f26c \ -RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ - rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist - -RAPIDS_REPOSITORY=rmm \ -RAPIDS_BUILD_TYPE=pull-request \ -RAPIDS_REF_NAME=1644 \ -RAPIDS_SHA=e93f26c \ -RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ - rapids-download-wheels-from-s3 python /tmp/libcudf_dist - -echo "librmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/librmm_*.whl)" >> /tmp/constraints.txt -echo "rmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/rmm_*.whl)" >> /tmp/constraints.txt -export PIP_CONSTRAINT=/tmp/constraints.txt -# --- end of section to remove ---# - ./ci/build_wheel.sh ${package_dir} RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 1317f5ce25c..54c4c7ae6ea 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -19,27 +19,6 @@ RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-f echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/libcudf_*.whl)" > /tmp/constraints.txt -# --- start of section to remove ---# -# TODO: remove this before merging -# use librmm and rmm from https://github.com/rapidsai/rmm/pull/1644 -RAPIDS_REPOSITORY=rmm \ -RAPIDS_BUILD_TYPE=pull-request \ -RAPIDS_REF_NAME=1644 \ -RAPIDS_SHA=e93f26c \ -RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ - rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist - -RAPIDS_REPOSITORY=rmm \ -RAPIDS_BUILD_TYPE=pull-request \ -RAPIDS_REF_NAME=1644 \ -RAPIDS_SHA=e93f26c \ -RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ - rapids-download-wheels-from-s3 python /tmp/libcudf_dist - -echo "librmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/librmm_*.whl)" >> /tmp/constraints.txt -echo "rmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/rmm_*.whl)" >> /tmp/constraints.txt -# --- end of section to remove ---# - export PIP_CONSTRAINT="/tmp/constraints.txt" ./ci/build_wheel.sh ${package_dir} diff --git a/ci/cudf_pandas_scripts/pandas-tests/run.sh b/ci/cudf_pandas_scripts/pandas-tests/run.sh index 55728425f95..2840ddc5839 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/run.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/run.sh @@ -14,32 +14,9 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-cudf-dep RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./local-cudf-dep -# --- start of section to remove ---# -# TODO: remove this before merging -# use librmm and rmm from https://github.com/rapidsai/rmm/pull/1644 -RAPIDS_REPOSITORY=rmm \ -RAPIDS_BUILD_TYPE=pull-request \ -RAPIDS_REF_NAME=1644 \ -RAPIDS_SHA=e93f26c \ -RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ - rapids-download-wheels-from-s3 cpp /tmp/local-rmm-dep - -RAPIDS_REPOSITORY=rmm \ -RAPIDS_BUILD_TYPE=pull-request \ -RAPIDS_REF_NAME=1644 \ -RAPIDS_SHA=e93f26c \ -RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ - rapids-download-wheels-from-s3 python /tmp/local-rmm-dep - -echo "librmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/local-rmm-dep/librmm_*.whl)" >> /tmp/constraints.txt -echo "rmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/local-rmm-dep/rmm_*.whl)" >> /tmp/constraints.txt - -export PIP_CONSTRAINT=/tmp/constraints.txt -# --- end of section to remove ---# - - -python -m pip install "$(echo ./local-cudf-dep/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" -python -m pip install --find-links $(pwd)/local-cudf-dep "$(echo ./local-cudf-dep/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test,pandas-tests]" +python -m pip install \ + "$(echo ./local-cudf-dep/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ + "$(echo ./local-cudf-dep/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test,pandas-tests]" RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index af3270f3bd5..1832ab829b0 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -39,31 +39,9 @@ else RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-cudf-dep RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./local-cudf-dep - # --- start of section to remove ---# - # TODO: remove this before merging - # use librmm and rmm from https://github.com/rapidsai/rmm/pull/1644 - RAPIDS_REPOSITORY=rmm \ - RAPIDS_BUILD_TYPE=pull-request \ - RAPIDS_REF_NAME=1644 \ - RAPIDS_SHA=e93f26c \ - RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ - rapids-download-wheels-from-s3 cpp /tmp/local-rmm-dep - - RAPIDS_REPOSITORY=rmm \ - RAPIDS_BUILD_TYPE=pull-request \ - RAPIDS_REF_NAME=1644 \ - RAPIDS_SHA=e93f26c \ - RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ - rapids-download-wheels-from-s3 python /tmp/local-rmm-dep - - echo "librmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/local-rmm-dep/librmm_*.whl)" >> /tmp/constraints.txt - echo "rmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/local-rmm-dep/rmm_*.whl)" >> /tmp/constraints.txt - - export PIP_CONSTRAINT=/tmp/constraints.txt - # --- end of section to remove ---# - - python -m pip install "$(echo ./local-cudf-dep/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" - python -m pip install --find-links $(pwd)/local-cudf-dep "$(echo ./local-cudf-dep/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test,cudf-pandas-tests]" + python -m pip install \ + "$(echo ./local-cudf-dep/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ + "$(echo ./local-cudf-dep/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test,cudf-pandas-tests]" fi python -m pytest -p cudf.pandas \ diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh index 0fbaf330f8c..c04651cbe33 100755 --- a/ci/test_wheel_cudf.sh +++ b/ci/test_wheel_cudf.sh @@ -8,8 +8,9 @@ RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-f RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist # echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" -python -m pip install --find-links $(pwd)/dist "$(echo ./dist/cudf*.whl)[test]" +python -m pip install \ + "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ + "$(echo ./dist/cudf*.whl)[test]" RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index c165b947f18..44ac0933bdc 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -23,12 +23,11 @@ RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist # echo to expand wildcard before adding `[extra]` requires for pip -rapids-logger "Install cudf wheel" -python -m pip install "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" -python -m pip install "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" - -rapids-logger "Install cudf_polars" -python -m pip install --find-links $(pwd)/dist "$(echo ./dist/cudf_polars*.whl)[test]" +rapids-logger "Install cudf, libcudf, and cudf_polars wheels" +python -m pip install \ + "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ + "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ + "$(echo ./dist/cudf_polars*.whl)[test]" RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ diff --git a/ci/test_wheel_dask_cudf.sh b/ci/test_wheel_dask_cudf.sh index 798ed8df062..2cf4b765522 100755 --- a/ci/test_wheel_dask_cudf.sh +++ b/ci/test_wheel_dask_cudf.sh @@ -9,10 +9,12 @@ RAPIDS_PY_WHEEL_NAME="dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE=" # Download the cudf built in the previous step RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-cudf-dep RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./local-cudf-dep -python -m pip install --no-deps --find-links $(pwd)/local-cudf-dep ./local-cudf-dep/cudf*.whl # echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install $(echo ./dist/dask_cudf*.whl)[test] +python -m pip install \ + "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ + "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ + "$(echo ./dist/dask_cudf*.whl)[test]" RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ diff --git a/dependencies.yaml b/dependencies.yaml index ccfa0931020..02ccbe46605 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -414,7 +414,7 @@ dependencies: common: - output_types: [conda] packages: - - &libcudf_conda libcudf==24.10.*,>=0.0.0a0 + - &libcudf_unsuffixed libcudf==24.10.*,>=0.0.0a0 specific: - output_types: [pyproject] matrices: @@ -428,7 +428,7 @@ dependencies: cuda_suffixed: "true" packages: - libcudf-cu11==24.10.*,>=0.0.0a0 - - {matrix: null, packages: [*libcudf_conda] } + - {matrix: null, packages: [*libcudf_unsuffixed] } cuda_version: specific: - output_types: conda From 226fdf00980569b1b1195686ac95012be7213868 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 15 Aug 2024 08:20:40 -0700 Subject: [PATCH 53/84] fix paths --- ci/test_wheel_dask_cudf.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/test_wheel_dask_cudf.sh b/ci/test_wheel_dask_cudf.sh index 2cf4b765522..dcd4e81307d 100755 --- a/ci/test_wheel_dask_cudf.sh +++ b/ci/test_wheel_dask_cudf.sh @@ -7,8 +7,8 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist # Download the cudf built in the previous step -RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-cudf-dep -RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./local-cudf-dep +RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist +RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install \ From 1117a736c1a8041f665ab166f4b0f1a25d054c1d Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 15 Aug 2024 17:33:05 -0500 Subject: [PATCH 54/84] remove unrelated changes (I think left over from a merge conflict resolution) --- ci/test_wheel_cudf_polars.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index 44ac0933bdc..448882f64c5 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -29,9 +29,6 @@ python -m pip install \ "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ "$(echo ./dist/cudf_polars*.whl)[test]" -RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} -RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ -mkdir -p "${RAPIDS_TESTS_DIR}" rapids-logger "Run cudf_polars tests" From f7b310ea9e6fc5290bd2269d24df7d45f6f8360b Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 16 Aug 2024 12:38:17 -0500 Subject: [PATCH 55/84] style --- python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt index 2984fc1d2f7..3ab6b9a33dc 100644 --- a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt +++ b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt @@ -53,8 +53,7 @@ rapids_cython_create_modules( LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX pylibcudf_ ASSOCIATED_TARGETS cudf ) -# link_to_pyarrow_headers(pylibcudf_interop) -# target_include_directories(pylibcudf_interop PUBLIC "$") +target_include_directories(pylibcudf_interop PUBLIC "$") include(${rapids-cmake-dir}/export/find_package_root.cmake) include(../../../../../cpp/cmake/thirdparty/get_nanoarrow.cmake) From 5e2a065967ab0a26abc2c5f8ecd60f1d65cf0ea7 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 19 Aug 2024 08:21:24 -0700 Subject: [PATCH 56/84] dependencies.yaml --- python/cudf/pyproject.toml | 1 - python/libcudf/pyproject.toml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 5137081312b..43c0e40c8e0 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -131,7 +131,6 @@ requires = [ "ninja", "numpy==1.23.*", "pyarrow==16.1.0.*", - "pylibcudf==24.10.*,>=0.0.0a0", "rmm==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index a02fc434637..56447a08aa9 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -39,6 +39,7 @@ classifiers = [ ] dependencies = [ "pyarrow>=16.1.0,<16.2.0a0", + "pylibcudf==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls] From 78af8e9f875aac2a153c51cbe5224c99a5e0ab52 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 19 Aug 2024 08:36:02 -0700 Subject: [PATCH 57/84] add build job, clean up formatting --- .github/workflows/build.yaml | 19 +++++++++++++++++++ .github/workflows/pr.yaml | 16 +++++++--------- dependencies.yaml | 2 +- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9943b02a521..3578cfede28 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -86,6 +86,25 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: pylibcudf + wheel-build-libcudf: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_cpp.sh + wheel-publish-pylibcudf: + needs: wheel-build-libcudf + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: libcudf wheel-build-cudf: needs: wheel-publish-pylibcudf secrets: inherit diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 73bfe53de9e..bdcd2a7a5bf 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -137,24 +137,22 @@ jobs: with: build_type: pull-request script: "ci/build_wheel_pylibcudf.sh" - wheel-build-python: + wheel-build-cudf: needs: [checks, wheel-build-cpp] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: pull-request script: "ci/build_wheel_python.sh" - wheel-tests-python: - needs: wheel-build-python + wheel-tests-cudf: + needs: wheel-build-cudf secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: pull-request script: ci/test_wheel_cudf.sh wheel-build-cudf-polars: - needs: - - wheel-build-pylibcudf - - wheel-build-cudf + needs: [wheel-build-pylibcudf, wheel-build-cudf] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: @@ -174,7 +172,7 @@ jobs: # pylibcudf or cudf_polars script: "ci/test_wheel_cudf_polars.sh" wheel-build-dask-cudf: - needs: wheel-build-python + needs: wheel-build-cudf secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: @@ -202,7 +200,7 @@ jobs: build-all -DBUILD_BENCHMARKS=ON --verbose; sccache -s; unit-tests-cudf-pandas: - needs: wheel-build-python + needs: wheel-build-cudf secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: @@ -212,7 +210,7 @@ jobs: script: ci/cudf_pandas_scripts/run_tests.sh pandas-tests: # run the Pandas unit tests using PR branch - needs: wheel-build-python + needs: wheel-build-cudf secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: diff --git a/dependencies.yaml b/dependencies.yaml index 81cdc928e55..3bf1ce64d27 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -506,7 +506,7 @@ dependencies: cuda_suffixed: "true" packages: - libcudf-cu11==24.10.*,>=0.0.0a0 - - {matrix: null, packages: [*libcudf_unsuffixed] } + - {matrix: null, packages: [*libcudf_unsuffixed]} cuda_version: specific: - output_types: conda From b33f421a7a552144b43e9809b38f1dbc3d3e49ee Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 19 Aug 2024 11:56:21 -0700 Subject: [PATCH 58/84] make pylibcudf depend on libcudf, use '_libcudf' instead of '_cpp' --- .github/workflows/build.yaml | 2 +- .github/workflows/pr.yaml | 6 +- ...ld_wheel_python.sh => build_wheel_cudf.sh} | 0 ci/build_wheel_libcudf.sh | 15 ++++ dependencies.yaml | 16 +++-- python/cudf/CMakeLists.txt | 2 +- python/libcudf/libcudf/load.py | 9 +-- python/pylibcudf/CMakeLists.txt | 68 ++----------------- python/pylibcudf/pylibcudf/__init__.py | 10 +++ python/pylibcudf/pyproject.toml | 2 + 10 files changed, 54 insertions(+), 76 deletions(-) rename ci/{build_wheel_python.sh => build_wheel_cudf.sh} (100%) create mode 100644 ci/build_wheel_libcudf.sh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 3578cfede28..09834affaa3 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -94,7 +94,7 @@ jobs: branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - script: ci/build_wheel_cpp.sh + script: ci/build_wheel_libcudf.sh wheel-publish-pylibcudf: needs: wheel-build-libcudf secrets: inherit diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index bdcd2a7a5bf..ad65d3fb55e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -122,14 +122,14 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:latest" run_script: "ci/build_docs.sh" - wheel-build-cpp: + wheel-build-libcudf: needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: pull-request - script: "ci/build_wheel_cpp.sh" + script: "ci/build_wheel_libcudf.sh" wheel-build-pylibcudf: needs: checks secrets: inherit @@ -138,7 +138,7 @@ jobs: build_type: pull-request script: "ci/build_wheel_pylibcudf.sh" wheel-build-cudf: - needs: [checks, wheel-build-cpp] + needs: [checks, wheel-build-libcudf] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_cudf.sh similarity index 100% rename from ci/build_wheel_python.sh rename to ci/build_wheel_cudf.sh diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh new file mode 100644 index 00000000000..9694c3f6144 --- /dev/null +++ b/ci/build_wheel_libcudf.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +set -euo pipefail + +package_dir="python/libcudf" + +./ci/build_wheel.sh ${package_dir} + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +mkdir -p ${package_dir}/final_dist +python -m auditwheel repair --exclude libarrow.so.1601 -w ${package_dir}/final_dist ${package_dir}/dist/* + +RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp ${package_dir}/final_dist diff --git a/dependencies.yaml b/dependencies.yaml index 3bf1ce64d27..a1acd9a0db9 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -140,14 +140,14 @@ files: includes: - test_python_common - test_python_cudf - py_rapids_build_pylibcudf: + py_build_pylibcudf: output: pyproject pyproject_dir: python/pylibcudf extras: table: build-system includes: - rapids_build_skbuild - py_build_pylibcudf: + py_rapids_build_pylibcudf: output: pyproject pyproject_dir: python/pylibcudf extras: @@ -392,6 +392,7 @@ dependencies: common: - output_types: conda packages: + - &libcudf_unsuffixed libcudf==24.10.*,>=0.0.0a0 - &rmm_unsuffixed rmm==24.10.*,>=0.0.0a0 - output_types: requirements packages: @@ -406,13 +407,15 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: + - &libcudf_cu12 libcudf-cu12==24.10.*,>=0.0.0a0 - rmm-cu12==24.10.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: + - &libcudf_cu11 libcudf-cu11==24.10.*,>=0.0.0a0 - rmm-cu11==24.10.*,>=0.0.0a0 - - {matrix: null, packages: [*rmm_unsuffixed]} + - {matrix: null, packages: [*libcudf_unsuffixed, *rmm_unsuffixed]} build_python_cudf: common: - output_types: conda @@ -492,7 +495,7 @@ dependencies: common: - output_types: [conda] packages: - - &libcudf_unsuffixed libcudf==24.10.*,>=0.0.0a0 + - *libcudf_unsuffixed specific: - output_types: [pyproject] matrices: @@ -696,6 +699,7 @@ dependencies: - typing_extensions>=4.0.0 - output_types: conda packages: + - *libcudf_unsuffixed - *rmm_unsuffixed - output_types: requirements packages: @@ -719,13 +723,15 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: + - *libcudf_cu12 - rmm-cu12==24.10.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: + - *libcudf_cu11 - rmm-cu11==24.10.*,>=0.0.0a0 - - {matrix: null, packages: [*rmm_unsuffixed]} + - {matrix: null, packages: [*libcudf_unsuffixed, *rmm_unsuffixed]} run_cudf: common: - output_types: [conda, requirements, pyproject] diff --git a/python/cudf/CMakeLists.txt b/python/cudf/CMakeLists.txt index a4634827931..3d6c58f629c 100644 --- a/python/cudf/CMakeLists.txt +++ b/python/cudf/CMakeLists.txt @@ -48,7 +48,7 @@ endif() # are installed to. Since there are multiple subpackages of cudf._lib that require access to # libcudf, we place the library and all its dependent artifacts in the cudf directory as a single # source of truth. -include(cmake/Modules/WheelHelpers.cmake) +include(../pylibcudf/cmake/Modules/WheelHelpers.cmake) # TODO: This install is currently overzealous. We should only install the libraries that are # downloaded by CPM during the build, not libraries that were found on the system. However, in diff --git a/python/libcudf/libcudf/load.py b/python/libcudf/libcudf/load.py index 00f5e046538..f6ba0d51bdb 100644 --- a/python/libcudf/libcudf/load.py +++ b/python/libcudf/libcudf/load.py @@ -31,10 +31,11 @@ def load_library(): except OSError: # If neither of these directories contain the library, we assume we are in an # environment where the C++ library is already installed somewhere else and the - # CMake build of the libcudf Python package was a no-op. Note that this approach - # won't work for real editable installs of the libcudf package, but that's not a - # use case I think we need to support. scikit-build-core has limited support for - # importlib.resources so there isn't a clean way to support that case yet. + # CMake build of the libcudf Python package was a no-op. + # + # Note that this approach won't work for real editable installs of the libcudf package. + # scikit-build-core has limited support for importlib.resources so there isn't a clean + # way to support that case yet. for lib_dir in ("lib", "lib64"): if os.path.isfile( lib := os.path.join( diff --git a/python/pylibcudf/CMakeLists.txt b/python/pylibcudf/CMakeLists.txt index 424d8372280..340ad120377 100644 --- a/python/pylibcudf/CMakeLists.txt +++ b/python/pylibcudf/CMakeLists.txt @@ -24,72 +24,16 @@ project( LANGUAGES CXX CUDA ) -option(FIND_CUDF_CPP "Search for existing CUDF C++ installations before defaulting to local files" - OFF -) -option(USE_LIBARROW_FROM_PYARROW "Only use the libarrow contained in pyarrow" OFF) -mark_as_advanced(USE_LIBARROW_FROM_PYARROW) - -# Find Python early so that later commands can use it -find_package(Python 3.9 REQUIRED COMPONENTS Interpreter) - -# If the user requested it we attempt to find CUDF. -if(FIND_CUDF_CPP) - include(rapids-cpm) - include(rapids-export) - include(rapids-find) - rapids_cpm_init() - - if(USE_LIBARROW_FROM_PYARROW) - # We need to find arrow before libcudf since libcudf requires it but doesn't bundle arrow - # libraries. These variables have no effect because we are always searching for arrow via - # pyarrow, but they must be set as they are required arguments to the function in - # get_arrow.cmake. - set(CUDF_USE_ARROW_STATIC OFF) - set(CUDF_ENABLE_ARROW_S3 OFF) - set(CUDF_ENABLE_ARROW_ORC OFF) - set(CUDF_ENABLE_ARROW_PYTHON OFF) - set(CUDF_ENABLE_ARROW_PARQUET OFF) - include(../../cpp/cmake/thirdparty/get_arrow.cmake) - endif() - - find_package(cudf "${RAPIDS_VERSION}" REQUIRED) +find_package(cudf "${RAPIDS_VERSION}" REQUIRED) - # an installed version of libcudf doesn't provide the dlpack headers so we need to download dlpack - # for the interop.pyx - include(../../cpp/cmake/thirdparty/get_dlpack.cmake) -else() - set(cudf_FOUND OFF) -endif() +# an installed version of libcudf doesn't provide the dlpack headers so we need to download dlpack +# for the interop.pyx +include(rapids-cpm) +rapids_cpm_init() +include(../../cpp/cmake/thirdparty/get_dlpack.cmake) include(rapids-cython-core) -if(NOT cudf_FOUND) - set(BUILD_TESTS OFF) - set(BUILD_BENCHMARKS OFF) - set(CUDF_BUILD_TESTUTIL OFF) - set(CUDF_BUILD_STREAMS_TEST_UTIL OFF) - set(CUDA_STATIC_RUNTIME ON) - - add_subdirectory(../../cpp cudf-cpp EXCLUDE_FROM_ALL) - - # libcudf targets are excluded by default above via EXCLUDE_FROM_ALL to remove extraneous - # components like headers from libcudacxx, but we do need the libraries. However, we want to - # control where they are installed to. Since there are multiple subpackages of pylibcudf that - # require access to libcudf, we place the library and all its dependent artifacts in the cudf - # directory as a single source of truth and modify the other rpaths appropriately. - set(cython_lib_dir pylibcudf) - include(cmake/Modules/WheelHelpers.cmake) - # TODO: This install is currently overzealous. We should only install the libraries that are - # downloaded by CPM during the build, not libraries that were found on the system. However, in - # practice right this would only be a problem is if libcudf was not found but some of the - # dependencies were, and we have no real use cases where that happens. - install_aliased_imported_targets( - TARGETS cudf arrow_shared nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp - DESTINATION ${cython_lib_dir} - ) -endif() - rapids_cython_init() include(cmake/Modules/LinkPyarrowHeaders.cmake) diff --git a/python/pylibcudf/pylibcudf/__init__.py b/python/pylibcudf/pylibcudf/__init__.py index 677fdaf80d0..e784c6c6dd5 100644 --- a/python/pylibcudf/pylibcudf/__init__.py +++ b/python/pylibcudf/pylibcudf/__init__.py @@ -1,5 +1,15 @@ # Copyright (c) 2023-2024, NVIDIA CORPORATION. +# If libcudf was installed as a wheel, we must request it to load the library symbols. +# Otherwise, we assume that the library was installed in a system path that ld can find. +try: + import libcudf +except ModuleNotFoundError: + pass +else: + libcudf.load_library() + del libcudf + from . import ( aggregation, binaryop, diff --git a/python/pylibcudf/pyproject.toml b/python/pylibcudf/pyproject.toml index b037508d03f..01c84e3fba2 100644 --- a/python/pylibcudf/pyproject.toml +++ b/python/pylibcudf/pyproject.toml @@ -19,6 +19,7 @@ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ "cuda-python>=11.7.1,<12.0a0", + "libcudf==24.10.*,>=0.0.0a0", "nvtx>=0.2.1", "packaging", "pyarrow>=16.1.0,<16.2.0a0", @@ -101,6 +102,7 @@ matrix-entry = "cuda_suffixed=true" requires = [ "cmake>=3.26.4,!=3.30.0", "cython>=3.0.3", + "libcudf==24.10.*,>=0.0.0a0", "ninja", "numpy==1.23.*", "pyarrow==16.1.0.*", From 32a39bfb63aa2a345997fcebdc4c16a2ad4e8955 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 19 Aug 2024 12:05:03 -0700 Subject: [PATCH 59/84] clean up build scripts --- ci/build_wheel_cpp.sh | 15 --------------- ci/build_wheel_cudf.sh | 16 +++++++++------- ci/build_wheel_pylibcudf.sh | 18 +++++++++++++++--- 3 files changed, 24 insertions(+), 25 deletions(-) delete mode 100755 ci/build_wheel_cpp.sh diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh deleted file mode 100755 index 9694c3f6144..00000000000 --- a/ci/build_wheel_cpp.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. - -set -euo pipefail - -package_dir="python/libcudf" - -./ci/build_wheel.sh ${package_dir} - -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" - -mkdir -p ${package_dir}/final_dist -python -m auditwheel repair --exclude libarrow.so.1601 -w ${package_dir}/final_dist ${package_dir}/dist/* - -RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp ${package_dir}/final_dist diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_cudf.sh index 54c4c7ae6ea..f3499953ed6 100755 --- a/ci/build_wheel_cudf.sh +++ b/ci/build_wheel_cudf.sh @@ -3,7 +3,6 @@ set -euo pipefail -package_name="cudf" package_dir="python/cudf" export SKBUILD_CMAKE_ARGS="-DUSE_LIBARROW_FROM_PYARROW=ON" @@ -16,14 +15,17 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" # Using env variable PIP_CONSTRAINT is necessary to ensure the constraints # are used when creating the isolated build environment. RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist - +RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python /tmp/libcudf_dist echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/libcudf_*.whl)" > /tmp/constraints.txt - +echo "pylibcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/pylibcudf_*.whl)" >> /tmp/constraints.txt export PIP_CONSTRAINT="/tmp/constraints.txt" + ./ci/build_wheel.sh ${package_dir} -cd ${package_dir} -mkdir -p final_dist -python -m auditwheel repair --exclude libcudf.so --exclude libarrow.so.1601 -w final_dist dist/* +python -m auditwheel repair \ + --exclude libcudf.so \ + --exclude libarrow.so.1601 \ + -w ${package_dir}/final_dist \ + ${package_dir}/dist/* -RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist +RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python ${package_dir}/final_dist diff --git a/ci/build_wheel_pylibcudf.sh b/ci/build_wheel_pylibcudf.sh index b25d118ff81..746cd21f442 100755 --- a/ci/build_wheel_pylibcudf.sh +++ b/ci/build_wheel_pylibcudf.sh @@ -7,10 +7,22 @@ package_dir="python/pylibcudf" export SKBUILD_CMAKE_ARGS="-DUSE_LIBARROW_FROM_PYARROW=ON" -./ci/build_wheel.sh ${package_dir} +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -python -m auditwheel repair -w ${package_dir}/final_dist ${package_dir}/dist/* +# Downloads libcudf wheel from this current build, +# then ensures 'cudf' wheel builds always use the 'libcudf' just built in the same CI run. +# +# Using env variable PIP_CONSTRAINT is necessary to ensure the constraints +# are used when creating the isolated build environment. +RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist +echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/libcudf_*.whl)" > /tmp/constraints.txt +export PIP_CONSTRAINT="/tmp/constraints.txt" +./ci/build_wheel.sh ${package_dir} + +python -m auditwheel repair \ + --exclude libcudf.so \ + -w ${package_dir}/final_dist \ + ${package_dir}/dist/* -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 ${package_dir}/final_dist From 74126f8f968e737f37257b437b9680e83d80bcbc Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 19 Aug 2024 12:26:05 -0700 Subject: [PATCH 60/84] more build script fixes --- .github/workflows/build.yaml | 18 +++++++++--------- .github/workflows/pr.yaml | 4 ++-- ci/build_wheel_pylibcudf.sh | 3 +-- ci/test_wheel_cudf.sh | 1 + 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 09834affaa3..fec77160410 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -67,7 +67,7 @@ jobs: node_type: "gpu-v100-latest-1" run_script: "ci/build_docs.sh" sha: ${{ inputs.sha }} - wheel-build-pylibcudf: + wheel-build-libcudf: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: @@ -75,9 +75,9 @@ jobs: branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - script: ci/build_wheel_pylibcudf.sh + script: ci/build_wheel_libcudf.sh wheel-publish-pylibcudf: - needs: wheel-build-pylibcudf + needs: wheel-build-libcudf secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 with: @@ -85,8 +85,8 @@ jobs: branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - package-name: pylibcudf - wheel-build-libcudf: + package-name: libcudf + wheel-build-pylibcudf: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: @@ -94,9 +94,9 @@ jobs: branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - script: ci/build_wheel_libcudf.sh + script: ci/build_wheel_pylibcudf.sh wheel-publish-pylibcudf: - needs: wheel-build-libcudf + needs: wheel-build-pylibcudf secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 with: @@ -104,9 +104,9 @@ jobs: branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - package-name: libcudf + package-name: pylibcudf wheel-build-cudf: - needs: wheel-publish-pylibcudf + needs: [wheel-publish-libcudf, wheel-publish-pylibcudf] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ad65d3fb55e..d1c6f7f8ac4 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -131,14 +131,14 @@ jobs: build_type: pull-request script: "ci/build_wheel_libcudf.sh" wheel-build-pylibcudf: - needs: checks + needs: [checks, wheel-build-libcudf] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: pull-request script: "ci/build_wheel_pylibcudf.sh" wheel-build-cudf: - needs: [checks, wheel-build-libcudf] + needs: [checks, wheel-build-libcudf, wheel-build-pylibcudf] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: diff --git a/ci/build_wheel_pylibcudf.sh b/ci/build_wheel_pylibcudf.sh index 746cd21f442..fa1721a5fab 100755 --- a/ci/build_wheel_pylibcudf.sh +++ b/ci/build_wheel_pylibcudf.sh @@ -5,8 +5,6 @@ set -euo pipefail package_dir="python/pylibcudf" -export SKBUILD_CMAKE_ARGS="-DUSE_LIBARROW_FROM_PYARROW=ON" - RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" # Downloads libcudf wheel from this current build, @@ -22,6 +20,7 @@ export PIP_CONSTRAINT="/tmp/constraints.txt" python -m auditwheel repair \ --exclude libcudf.so \ + --exclude libarrow.so.1601 \ -w ${package_dir}/final_dist \ ${package_dir}/dist/* diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh index a399a9ba94b..163b79fbc1f 100755 --- a/ci/test_wheel_cudf.sh +++ b/ci/test_wheel_cudf.sh @@ -19,6 +19,7 @@ RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ mkdir -p "${RAPIDS_TESTS_DIR}" + rapids-logger "pytest pylibcudf" pushd python/pylibcudf/pylibcudf/tests python -m pytest \ From d64f493759c0cd554a30cf90e2893b7320f62305 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 19 Aug 2024 12:40:09 -0700 Subject: [PATCH 61/84] make build script executable --- ci/build_wheel_cudf.sh | 2 -- ci/build_wheel_libcudf.sh | 0 2 files changed, 2 deletions(-) mode change 100644 => 100755 ci/build_wheel_libcudf.sh diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_cudf.sh index f3499953ed6..ca6325b6ddd 100755 --- a/ci/build_wheel_cudf.sh +++ b/ci/build_wheel_cudf.sh @@ -5,8 +5,6 @@ set -euo pipefail package_dir="python/cudf" -export SKBUILD_CMAKE_ARGS="-DUSE_LIBARROW_FROM_PYARROW=ON" - RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" # Downloads libcudf wheel from this current build, diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh old mode 100644 new mode 100755 From af82d2785a887e944005c497cd9fb3689998334a Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 19 Aug 2024 13:46:07 -0700 Subject: [PATCH 62/84] remove ncomp and arrow vendoring --- .github/workflows/build.yaml | 2 +- python/cudf/CMakeLists.txt | 16 ---------------- python/libcudf/CMakeLists.txt | 10 ++++++++++ .../cmake/Modules/WheelHelpers.cmake | 0 4 files changed, 11 insertions(+), 17 deletions(-) rename python/{pylibcudf => libcudf}/cmake/Modules/WheelHelpers.cmake (100%) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index fec77160410..f5eda4ab65b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -76,7 +76,7 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} script: ci/build_wheel_libcudf.sh - wheel-publish-pylibcudf: + wheel-publish-libcudf: needs: wheel-build-libcudf secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 diff --git a/python/cudf/CMakeLists.txt b/python/cudf/CMakeLists.txt index 3d6c58f629c..72f20b30052 100644 --- a/python/cudf/CMakeLists.txt +++ b/python/cudf/CMakeLists.txt @@ -42,19 +42,3 @@ add_subdirectory(udf_cpp) if(DEFINED cython_lib_dir) rapids_cython_add_rpath_entries(TARGET cudf PATHS "${cython_lib_dir}") endif() - -# libcudf targets are excluded by default above via EXCLUDE_FROM_ALL to remove extraneous components -# like headers from libcudacxx, but we do need the libraries. However, we want to control where they -# are installed to. Since there are multiple subpackages of cudf._lib that require access to -# libcudf, we place the library and all its dependent artifacts in the cudf directory as a single -# source of truth. -include(../pylibcudf/cmake/Modules/WheelHelpers.cmake) - -# TODO: This install is currently overzealous. We should only install the libraries that are -# downloaded by CPM during the build, not libraries that were found on the system. However, in -# practice this would only be a problem if libcudf was not found but some of the dependencies were, -# and we have no real use cases where that happens. -install_aliased_imported_targets( - TARGETS cudf arrow_shared nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp - DESTINATION ${cython_lib_dir} -) diff --git a/python/libcudf/CMakeLists.txt b/python/libcudf/CMakeLists.txt index bf2d6e8fcf0..db3d3716231 100644 --- a/python/libcudf/CMakeLists.txt +++ b/python/libcudf/CMakeLists.txt @@ -47,3 +47,13 @@ set(CUDA_STATIC_RUNTIME ON) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) add_subdirectory(../../cpp cudf-cpp) + +# Ensure other libraries needed by libcudf.so get installed alongside it. +# +# TODO: This install is currently overzealous. We should only install the libraries that are +# downloaded by CPM during the build, not libraries that were found on the system. +include(cmake/Modules/WheelHelpers.cmake) +install_aliased_imported_targets( + TARGETS cudf arrow_shared nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp + DESTINATION ${CMAKE_LIBRARY_OUTPUT_DIRECTORY} +) diff --git a/python/pylibcudf/cmake/Modules/WheelHelpers.cmake b/python/libcudf/cmake/Modules/WheelHelpers.cmake similarity index 100% rename from python/pylibcudf/cmake/Modules/WheelHelpers.cmake rename to python/libcudf/cmake/Modules/WheelHelpers.cmake From f776095d25dcbfff9375f4d407ad04c321e36eaf Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 19 Aug 2024 13:47:56 -0700 Subject: [PATCH 63/84] more auditwheel --- ci/build_wheel_cudf.sh | 3 +++ ci/build_wheel_pylibcudf.sh | 3 +++ 2 files changed, 6 insertions(+) diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_cudf.sh index ca6325b6ddd..3e53c97ec0b 100755 --- a/ci/build_wheel_cudf.sh +++ b/ci/build_wheel_cudf.sh @@ -23,6 +23,9 @@ export PIP_CONSTRAINT="/tmp/constraints.txt" python -m auditwheel repair \ --exclude libcudf.so \ --exclude libarrow.so.1601 \ + --exclude libnvcomp.so \ + --exclude libnvcomp_bitcomp.so \ + --exclude libnvcomp_gdeflate.so \ -w ${package_dir}/final_dist \ ${package_dir}/dist/* diff --git a/ci/build_wheel_pylibcudf.sh b/ci/build_wheel_pylibcudf.sh index fa1721a5fab..d0468455027 100755 --- a/ci/build_wheel_pylibcudf.sh +++ b/ci/build_wheel_pylibcudf.sh @@ -21,6 +21,9 @@ export PIP_CONSTRAINT="/tmp/constraints.txt" python -m auditwheel repair \ --exclude libcudf.so \ --exclude libarrow.so.1601 \ + --exclude libnvcomp.so \ + --exclude libnvcomp_bitcomp.so \ + --exclude libnvcomp_gdeflate.so \ -w ${package_dir}/final_dist \ ${package_dir}/dist/* From 796fc1d608f394a53d57a9170989205be09998ab Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 19 Aug 2024 14:35:05 -0700 Subject: [PATCH 64/84] add Arrow linking --- python/libcudf/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/libcudf/CMakeLists.txt b/python/libcudf/CMakeLists.txt index db3d3716231..823fa4c265a 100644 --- a/python/libcudf/CMakeLists.txt +++ b/python/libcudf/CMakeLists.txt @@ -46,6 +46,8 @@ set(CUDA_STATIC_RUNTIME ON) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) +include(../pylibcudf/cmake/Modules/LinkPyarrowHeaders.cmake) + add_subdirectory(../../cpp cudf-cpp) # Ensure other libraries needed by libcudf.so get installed alongside it. From d6eba06260d58e6012eca2251665f63822e29350 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 19 Aug 2024 14:36:08 -0700 Subject: [PATCH 65/84] revert logger visibility fix (should no longer be necessary as of https://github.com/rapidsai/rmm/issues/1645 and https://github.com/rapidsai/cudf/pull/15982) --- cpp/include/cudf/utilities/logger.hpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/cpp/include/cudf/utilities/logger.hpp b/cpp/include/cudf/utilities/logger.hpp index 9abf3a92f6c..45d5d1b12e1 100644 --- a/cpp/include/cudf/utilities/logger.hpp +++ b/cpp/include/cudf/utilities/logger.hpp @@ -43,11 +43,6 @@ namespace CUDF_EXPORT cudf { * * @return spdlog::logger& The logger. */ -// This is a workaround for Doxygen not being able to parse the `visibility` attribute -/// \cond DOXYGEN_IGNORE -// We need this symbol to be visible because this TU is marked as having all -// symbols hidden by default in cpp/CMakelists.txt. -__attribute__((visibility("default"))) spdlog::logger& logger(); -/// \endcond +spdlog::logger& logger(); } // namespace CUDF_EXPORT cudf From cb2bf5ace3cee300cbb05e59c96c5052f7428184 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 19 Aug 2024 15:14:35 -0700 Subject: [PATCH 66/84] add 'cpp' identifier for rapids-download-wheels-from-s3 calls --- ci/cudf_pandas_scripts/pandas-tests/run.sh | 2 +- ci/cudf_pandas_scripts/run_tests.sh | 2 +- ci/test_wheel_cudf.sh | 2 +- ci/test_wheel_cudf_polars.sh | 2 +- ci/test_wheel_dask_cudf.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/cudf_pandas_scripts/pandas-tests/run.sh b/ci/cudf_pandas_scripts/pandas-tests/run.sh index a51fa3e4376..9a689f3f3fe 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/run.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/run.sh @@ -14,7 +14,7 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" # Download the cudf, libcudf, and pylibcudf built in the previous step RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist # echo to expand wildcard before adding `[extra]` requires for pip diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index cf3c7e898ae..834045bf5ed 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -39,7 +39,7 @@ else # Download the cudf, libcudf, and pylibcudf built in the previous step RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist - RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist + RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist # echo to expand wildcard before adding `[extra]` requires for pip diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh index 7aa3d41326f..8e340d800ce 100755 --- a/ci/test_wheel_cudf.sh +++ b/ci/test_wheel_cudf.sh @@ -7,7 +7,7 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" # Download the cudf, libcudf, and pylibcudf built in the previous step RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist # echo to expand wildcard before adding `[extra]` requires for pip diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index 5ec541da7b4..94defb9d7af 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -22,7 +22,7 @@ RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE # Download the cudf, libcudf, and pylibcudf built in the previous step RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist rapids-logger "Installing cudf_polars and its dependencies" diff --git a/ci/test_wheel_dask_cudf.sh b/ci/test_wheel_dask_cudf.sh index 108042df188..11669e9b647 100755 --- a/ci/test_wheel_dask_cudf.sh +++ b/ci/test_wheel_dask_cudf.sh @@ -8,7 +8,7 @@ RAPIDS_PY_WHEEL_NAME="dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE=" # Download the cudf, libcudf, and pylibcudf built in the previous step RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist # echo to expand wildcard before adding `[extra]` requires for pip From 26dde4274b05ce0f74b623c26463421bf842c6d6 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 19 Aug 2024 15:41:30 -0700 Subject: [PATCH 67/84] more dependencies.yaml changes --- dependencies.yaml | 108 ++++++++++++++++++++++++++-------- python/libcudf/pyproject.toml | 1 - 2 files changed, 82 insertions(+), 27 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 58a482cea08..718032f185f 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -10,6 +10,7 @@ files: - build_all - build_cpp - build_python_common + - build_python_libcudf - build_python_pylibcudf - build_python_cudf - cuda @@ -80,26 +81,13 @@ files: - docs - libarrow_run - py_version - # This is the shared library, bundled as a wheel. It is meant to be consumed by the wrapper. - py_build_libcudf: + py_build_cudf: output: pyproject - pyproject_dir: python/libcudf + pyproject_dir: python/cudf extras: table: build-system includes: - rapids_build_skbuild - # This is the wrapper that gets used in Python, not the shared library wheel - py_rapids_build_libcudf: - output: pyproject - pyproject_dir: python/libcudf - extras: - table: tool.rapids-build-backend - key: requires - includes: - - build_base - - build_cpp - - build_python_common - - build_python_cudf py_rapids_build_cudf: output: pyproject pyproject_dir: python/cudf @@ -110,13 +98,6 @@ files: - build_base - build_python_common - build_python_cudf - py_run_libcudf: - output: pyproject - pyproject_dir: python/libcudf - extras: - table: project - includes: - - pyarrow_run py_run_cudf: output: pyproject pyproject_dir: python/cudf @@ -127,7 +108,7 @@ files: - run_cudf - pyarrow_run - depends_on_cupy - - run_libcudf + - depends_on_libcudf - depends_on_pylibcudf py_test_cudf: output: pyproject @@ -138,6 +119,31 @@ files: includes: - test_python_common - test_python_cudf + py_build_libcudf: + output: pyproject + pyproject_dir: python/libcudf + extras: + table: build-system + includes: + - rapids_build_skbuild + py_rapids_build_libcudf: + output: pyproject + pyproject_dir: python/libcudf + extras: + table: tool.rapids-build-backend + key: requires + includes: + - build_base + - build_cpp + - build_python_common + - build_python_libcudf + py_run_libcudf: + output: pyproject + pyproject_dir: python/libcudf + extras: + table: project + includes: + - pyarrow_run py_build_pylibcudf: output: pyproject pyproject_dir: python/pylibcudf @@ -386,12 +392,37 @@ dependencies: # Sync with conda build constraint & wheel run constraint. # TODO: Change to `2.0.*` for NumPy 2 - numpy==1.23.* + build_python_libcudf: + common: + - output_types: conda + packages: + - &rmm_unsuffixed rmm==24.10.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + # This index is needed for rmm-cu{11,12}. + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - &rmm_cu12 rmm-cu12==24.10.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - &rmm_cu11 rmm-cu11==24.10.*,>=0.0.0a0 + - {matrix: null, packages: [*rmm_unsuffixed]} build_python_pylibcudf: common: - output_types: conda packages: - &libcudf_unsuffixed libcudf==24.10.*,>=0.0.0a0 - - &rmm_unsuffixed rmm==24.10.*,>=0.0.0a0 + - *rmm_unsuffixed - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -406,13 +437,13 @@ dependencies: cuda_suffixed: "true" packages: - &libcudf_cu12 libcudf-cu12==24.10.*,>=0.0.0a0 - - &rmm_cu12 rmm-cu12==24.10.*,>=0.0.0a0 + - *rmm_cu12 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - &libcudf_cu11 libcudf-cu11==24.10.*,>=0.0.0a0 - - &rmm_cu11 rmm-cu11==24.10.*,>=0.0.0a0 + - *rmm_cu11 - {matrix: null, packages: [*libcudf_unsuffixed, *rmm_unsuffixed]} build_python_cudf: common: @@ -886,6 +917,31 @@ dependencies: packages: - dask-cuda==24.10.*,>=0.0.0a0 - *numba + depends_on_libcudf: + common: + - output_types: conda + packages: + - *libcudf_unsuffixed + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + # This index is needed for rmm, cubinlinker, ptxcompiler. + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - *libcudf_cu12 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - *libcudf_cu11 + - {matrix: null, packages: [*libcudf_unsuffixed]} depends_on_pylibcudf: common: - output_types: conda diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index 3e853c6926a..a02fc434637 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -73,6 +73,5 @@ requires = [ "ninja", "numpy==1.23.*", "pyarrow==16.1.0.*", - "pylibcudf==24.10.*,>=0.0.0a0", "rmm==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From 2fa4935aaa8637befa178c9aeceb5e31579654b6 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 19 Aug 2024 15:45:19 -0700 Subject: [PATCH 68/84] add libcudf build dependency to cudf --- dependencies.yaml | 4 ++++ python/cudf/pyproject.toml | 1 + 2 files changed, 5 insertions(+) diff --git a/dependencies.yaml b/dependencies.yaml index 718032f185f..3f653fda33c 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -449,6 +449,7 @@ dependencies: common: - output_types: conda packages: + - *libcudf_unsuffixed - *rmm_unsuffixed - pip - pip: @@ -467,16 +468,19 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: + - *libcudf_cu12 - &pylibcudf_cu12 pylibcudf-cu12==24.10.*,>=0.0.0a0 - *rmm_cu12 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: + - *libcudf_cu11 - &pylibcudf_cu11 pylibcudf-cu11==24.10.*,>=0.0.0a0 - *rmm_cu11 - matrix: packages: + - *libcudf_unsuffixed - &pylibcudf_unsuffixed pylibcudf==24.10.*,>=0.0.0a0 - *rmm_unsuffixed libarrow_build: diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 15e3716c014..5137081312b 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -127,6 +127,7 @@ matrix-entry = "cuda_suffixed=true" requires = [ "cmake>=3.26.4,!=3.30.0", "cython>=3.0.3", + "libcudf==24.10.*,>=0.0.0a0", "ninja", "numpy==1.23.*", "pyarrow==16.1.0.*", From 0661e63e7ff2136e54c5e5cedc624efae11355a7 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 20 Aug 2024 10:16:25 -0500 Subject: [PATCH 69/84] Apply suggestions from code review Co-authored-by: Bradley Dice --- .github/workflows/build.yaml | 1 + ci/test_wheel_dask_cudf.sh | 2 +- python/libcudf/CMakeLists.txt | 4 ++-- python/libcudf/libcudf/_version.py | 2 +- python/libcudf/pyproject.toml | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f5eda4ab65b..263a841fb37 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -87,6 +87,7 @@ jobs: date: ${{ inputs.date }} package-name: libcudf wheel-build-pylibcudf: + needs: [wheel-publish-libcudf] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: diff --git a/ci/test_wheel_dask_cudf.sh b/ci/test_wheel_dask_cudf.sh index 11669e9b647..c04769b46b8 100755 --- a/ci/test_wheel_dask_cudf.sh +++ b/ci/test_wheel_dask_cudf.sh @@ -15,7 +15,7 @@ RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels python -m pip install \ "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ "$(echo ./dist/dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \ - "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \ + "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} diff --git a/python/libcudf/CMakeLists.txt b/python/libcudf/CMakeLists.txt index 823fa4c265a..05bca21ec57 100644 --- a/python/libcudf/CMakeLists.txt +++ b/python/libcudf/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -36,7 +36,7 @@ unset(cudf_FOUND) set(USE_LIBARROW_FROM_PYARROW ON) # Find Python early so that later commands can use it -find_package(Python 3.9 REQUIRED COMPONENTS Interpreter) +find_package(Python 3.10 REQUIRED COMPONENTS Interpreter) set(BUILD_TESTS OFF) set(BUILD_BENCHMARKS OFF) diff --git a/python/libcudf/libcudf/_version.py b/python/libcudf/libcudf/_version.py index 7dd732b4905..dac9d074fa7 100644 --- a/python/libcudf/libcudf/_version.py +++ b/python/libcudf/libcudf/_version.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index a02fc434637..b74f1db5b12 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -28,7 +28,7 @@ authors = [ { name = "NVIDIA Corporation" }, ] license = { text = "Apache 2.0" } -requires-python = ">=3.9" +requires-python = ">=3.10" classifiers = [ "Intended Audience :: Developers", "Topic :: Database", From 0ba7a460305fa5517451ed1bc71a51515303a4e4 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 20 Aug 2024 08:30:11 -0700 Subject: [PATCH 70/84] no Cython or Numpy deps --- dependencies.yaml | 20 +++++++++++--------- python/libcudf/libcudf/_version.py | 2 +- python/libcudf/pyproject.toml | 4 +--- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 3f653fda33c..09b31d00420 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -135,7 +135,6 @@ files: includes: - build_base - build_cpp - - build_python_common - build_python_libcudf py_run_libcudf: output: pyproject @@ -385,7 +384,7 @@ dependencies: - cython>=3.0.3 # Hard pin the patch version used during the build. This must be kept # in sync with the version pinned in get_arrow.cmake. - - pyarrow==16.1.0.* + - &pyarrow_build pyarrow==16.1.0.* - output_types: pyproject packages: # Hard pin the patch version used during the build. @@ -396,7 +395,10 @@ dependencies: common: - output_types: conda packages: - - &rmm_unsuffixed rmm==24.10.*,>=0.0.0a0 + - &librmm_unsuffixed librmm==24.10.*,>=0.0.0a0 + - output_types: [conda, requirements, pyproject] + packages: + - *pyarrow_build - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -410,19 +412,19 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - &rmm_cu12 rmm-cu12==24.10.*,>=0.0.0a0 + - librmm-cu12==24.10.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - &rmm_cu11 rmm-cu11==24.10.*,>=0.0.0a0 - - {matrix: null, packages: [*rmm_unsuffixed]} + - librmm-cu11==24.10.*,>=0.0.0a0 + - {matrix: null, packages: [*librmm_unsuffixed]} build_python_pylibcudf: common: - output_types: conda packages: - &libcudf_unsuffixed libcudf==24.10.*,>=0.0.0a0 - - *rmm_unsuffixed + - &rmm_unsuffixed rmm==24.10.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -437,13 +439,13 @@ dependencies: cuda_suffixed: "true" packages: - &libcudf_cu12 libcudf-cu12==24.10.*,>=0.0.0a0 - - *rmm_cu12 + - &rmm_cu12 rmm-cu12==24.10.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - &libcudf_cu11 libcudf-cu11==24.10.*,>=0.0.0a0 - - *rmm_cu11 + - &rmm_cu11 rmm-cu11==24.10.*,>=0.0.0a0 - {matrix: null, packages: [*libcudf_unsuffixed, *rmm_unsuffixed]} build_python_cudf: common: diff --git a/python/libcudf/libcudf/_version.py b/python/libcudf/libcudf/_version.py index dac9d074fa7..7dd732b4905 100644 --- a/python/libcudf/libcudf/_version.py +++ b/python/libcudf/libcudf/_version.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index b74f1db5b12..fd01f7f6e2f 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -69,9 +69,7 @@ dependencies-file = "../../dependencies.yaml" matrix-entry = "cuda_suffixed=true" requires = [ "cmake>=3.26.4,!=3.30.0", - "cython>=3.0.3", + "librmm==24.10.*,>=0.0.0a0", "ninja", - "numpy==1.23.*", "pyarrow==16.1.0.*", - "rmm==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From fbd1b56af3ea9b73a8ad0a7e472f6fdc5bf9f9ee Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 20 Aug 2024 08:35:44 -0700 Subject: [PATCH 71/84] simplify dependencies.yaml --- dependencies.yaml | 26 ++------------------------ 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 09b31d00420..d42f27af1b8 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -24,7 +24,6 @@ files: - rapids_build_setuptools - run_common - run_cudf - - run_libcudf - run_pylibcudf - run_dask_cudf - run_custreamz @@ -166,6 +165,7 @@ files: extras: table: project includes: + - depends_on_libcudf - run_pylibcudf - pyarrow_run py_test_pylibcudf: @@ -510,25 +510,6 @@ dependencies: packages: # Allow runtime version to float up to patch version - pyarrow>=16.1.0,<16.2.0a0 - run_libcudf: - common: - - output_types: [conda] - packages: - - *libcudf_unsuffixed - specific: - - output_types: [pyproject] - matrices: - - matrix: - cuda: "12.*" - cuda_suffixed: "true" - packages: - - libcudf-cu12==24.10.*,>=0.0.0a0 - - matrix: - cuda: "11.*" - cuda_suffixed: "true" - packages: - - libcudf-cu11==24.10.*,>=0.0.0a0 - - {matrix: null, packages: [*libcudf_unsuffixed]} cuda_version: specific: - output_types: conda @@ -718,7 +699,6 @@ dependencies: - typing_extensions>=4.0.0 - output_types: conda packages: - - *libcudf_unsuffixed - *rmm_unsuffixed - output_types: requirements packages: @@ -742,15 +722,13 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - *libcudf_cu12 - *rmm_cu12 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - *libcudf_cu11 - *rmm_cu11 - - {matrix: null, packages: [*libcudf_unsuffixed, *rmm_unsuffixed]} + - {matrix: null, packages: [*rmm_unsuffixed]} run_cudf: common: - output_types: [conda, requirements, pyproject] From 707a8bd50a8e0a354edd145f98f222c2be571f25 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 20 Aug 2024 09:46:01 -0700 Subject: [PATCH 72/84] more fiddling with dependencies.yaml --- .../all_cuda-118_arch-x86_64.yaml | 1 - .../all_cuda-125_arch-x86_64.yaml | 1 - dependencies.yaml | 61 ++++++++----------- 3 files changed, 27 insertions(+), 36 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 5143515e4d1..d0d18e57abc 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -40,7 +40,6 @@ dependencies: - libarrow-acero==16.1.0.* - libarrow-dataset==16.1.0.* - libarrow==16.1.0.* -- libcudf==24.10.*,>=0.0.0a0 - libcufile-dev=1.4.0.31 - libcufile=1.4.0.31 - libcurand-dev=10.3.0.86 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 923a6e0e82f..caf39a32d79 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -41,7 +41,6 @@ dependencies: - libarrow-acero==16.1.0.* - libarrow-dataset==16.1.0.* - libarrow==16.1.0.* -- libcudf==24.10.*,>=0.0.0a0 - libcufile-dev - libcurand-dev - libkvikio==24.10.*,>=0.0.0a0 diff --git a/dependencies.yaml b/dependencies.yaml index d42f27af1b8..0fa4cc6c504 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -15,6 +15,7 @@ files: - build_python_cudf - cuda - cuda_version + - depends_on_librmm - develop - docs - libarrow_build @@ -97,6 +98,7 @@ files: - build_base - build_python_common - build_python_cudf + - depends_on_libcudf py_run_cudf: output: pyproject pyproject_dir: python/cudf @@ -135,6 +137,7 @@ files: - build_base - build_cpp - build_python_libcudf + - depends_on_librmm py_run_libcudf: output: pyproject pyproject_dir: python/libcudf @@ -159,6 +162,7 @@ files: - build_base - build_python_common - build_python_pylibcudf + - depends_on_libcudf py_run_pylibcudf: output: pyproject pyproject_dir: python/pylibcudf @@ -393,37 +397,13 @@ dependencies: - numpy==1.23.* build_python_libcudf: common: - - output_types: conda - packages: - - &librmm_unsuffixed librmm==24.10.*,>=0.0.0a0 - output_types: [conda, requirements, pyproject] packages: - *pyarrow_build - - output_types: requirements - packages: - # pip recognizes the index as a global option for the requirements.txt file - # This index is needed for rmm-cu{11,12}. - - --extra-index-url=https://pypi.nvidia.com - - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple - specific: - - output_types: [requirements, pyproject] - matrices: - - matrix: - cuda: "12.*" - cuda_suffixed: "true" - packages: - - librmm-cu12==24.10.*,>=0.0.0a0 - - matrix: - cuda: "11.*" - cuda_suffixed: "true" - packages: - - librmm-cu11==24.10.*,>=0.0.0a0 - - {matrix: null, packages: [*librmm_unsuffixed]} build_python_pylibcudf: common: - output_types: conda packages: - - &libcudf_unsuffixed libcudf==24.10.*,>=0.0.0a0 - &rmm_unsuffixed rmm==24.10.*,>=0.0.0a0 - output_types: requirements packages: @@ -438,20 +418,17 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - &libcudf_cu12 libcudf-cu12==24.10.*,>=0.0.0a0 - &rmm_cu12 rmm-cu12==24.10.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - &libcudf_cu11 libcudf-cu11==24.10.*,>=0.0.0a0 - &rmm_cu11 rmm-cu11==24.10.*,>=0.0.0a0 - - {matrix: null, packages: [*libcudf_unsuffixed, *rmm_unsuffixed]} + - {matrix: null, packages: [*rmm_unsuffixed]} build_python_cudf: common: - output_types: conda packages: - - *libcudf_unsuffixed - *rmm_unsuffixed - pip - pip: @@ -470,19 +447,16 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - *libcudf_cu12 - &pylibcudf_cu12 pylibcudf-cu12==24.10.*,>=0.0.0a0 - *rmm_cu12 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - *libcudf_cu11 - &pylibcudf_cu11 pylibcudf-cu11==24.10.*,>=0.0.0a0 - *rmm_cu11 - matrix: packages: - - *libcudf_unsuffixed - &pylibcudf_unsuffixed pylibcudf==24.10.*,>=0.0.0a0 - *rmm_unsuffixed libarrow_build: @@ -905,7 +879,7 @@ dependencies: common: - output_types: conda packages: - - *libcudf_unsuffixed + - &libcudf_unsuffixed libcudf==24.10.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -919,12 +893,12 @@ dependencies: cuda: "12.*" cuda_suffixed: "true" packages: - - *libcudf_cu12 + - libcudf-cu12==24.10.*,>=0.0.0a0 - matrix: cuda: "11.*" cuda_suffixed: "true" packages: - - *libcudf_cu11 + - libcudf-cu11==24.10.*,>=0.0.0a0 - {matrix: null, packages: [*libcudf_unsuffixed]} depends_on_pylibcudf: common: @@ -1016,6 +990,25 @@ dependencies: packages: &cupy_packages_cu11 - cupy-cuda11x>=12.0.0 - {matrix: null, packages: *cupy_packages_cu11} + depends_on_librmm: + common: + - output_types: conda + packages: + - &librmm_unsuffixed librmm==24.10.*,>=0.0.0a0 + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - librmm-cu12==24.10.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - librmm-cu11==24.10.*,>=0.0.0a0 + - {matrix: null, packages: [*librmm_unsuffixed]} test_python_pandas_cudf: common: - output_types: [requirements, pyproject] From 3a939f4eae7bf62b262b30f8ee7ed6d57267b5f1 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 20 Aug 2024 09:52:44 -0700 Subject: [PATCH 73/84] standardize comments --- build.sh | 3 ++- ci/build_wheel_cudf.sh | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/build.sh b/build.sh index adc858ed160..18fe5569aac 100755 --- a/build.sh +++ b/build.sh @@ -342,6 +342,7 @@ if buildAll || hasArg libcudf; then fi fi +# Build and install the libcudf Python package if buildAll || hasArg libcudfwheel; then cd ${REPODIR}/python/libcudf @@ -357,7 +358,7 @@ if buildAll || hasArg pylibcudf; then python ${PYTHON_ARGS_FOR_INSTALL} . fi -# Build and install the cudf Python packages +# Build and install the cudf Python package if buildAll || hasArg cudf; then cd ${REPODIR}/python/cudf diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_cudf.sh index 3e53c97ec0b..0a4c2e613c6 100755 --- a/ci/build_wheel_cudf.sh +++ b/ci/build_wheel_cudf.sh @@ -7,8 +7,8 @@ package_dir="python/cudf" RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# Downloads libcudf wheel from this current build, -# then ensures 'cudf' wheel builds always use the 'libcudf' just built in the same CI run. +# Downloads libcudf and pylibcudf wheels from this current build, +# then ensures 'cudf' wheel builds always use the 'libcudf' and 'pylibcudf' just built in the same CI run. # # Using env variable PIP_CONSTRAINT is necessary to ensure the constraints # are used when creating the isolated build environment. From 3cbbc5b2c78d040f8690003a7aac86f031ea4e20 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 20 Aug 2024 13:02:52 -0700 Subject: [PATCH 74/84] test older rapids-cmake --- rapids_config.cmake | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/rapids_config.cmake b/rapids_config.cmake index 3a88769f6e7..bfbaaeebd1c 100644 --- a/rapids_config.cmake +++ b/rapids_config.cmake @@ -26,11 +26,14 @@ else() ) endif() -if(NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/CUDF_RAPIDS-${RAPIDS_VERSION_MAJOR_MINOR}.cmake") - file( - DOWNLOAD - "https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-${RAPIDS_VERSION_MAJOR_MINOR}/RAPIDS.cmake" - "${CMAKE_CURRENT_BINARY_DIR}/CUDF_RAPIDS-${RAPIDS_VERSION_MAJOR_MINOR}.cmake" - ) -endif() +include(FetchContent) +# GIT_TAG = last commit prior to https://github.com/rapidsai/rapids-cmake/pull/678 +FetchContent_Declare( + rapids-cmake + GIT_REPOSITORY https://raw.githubusercontent.com/rapidsai/rapids-cmake.git + GIT_TAG 0e65a8481b9895b5075d2b601690540a21b45deb +) +file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-24.10/RAPIDS.cmake + ${CMAKE_CURRENT_BINARY_DIR}/CUDF_RAPIDS-${RAPIDS_VERSION_MAJOR_MINOR}.cmake) + include("${CMAKE_CURRENT_BINARY_DIR}/CUDF_RAPIDS-${RAPIDS_VERSION_MAJOR_MINOR}.cmake") From 97a908bd852c5268064e9ef87101e6d61d1af618 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 20 Aug 2024 13:07:16 -0700 Subject: [PATCH 75/84] grrrrrr cmakelint --- ci/check_style.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/check_style.sh b/ci/check_style.sh index 634d8b0d702..b7b8164cf4f 100755 --- a/ci/check_style.sh +++ b/ci/check_style.sh @@ -1,6 +1,8 @@ #!/bin/bash # Copyright (c) 2020-2024, NVIDIA CORPORATION. +exit 0 + set -euo pipefail rapids-logger "Create checks conda environment" From eb4391b5a9ab599ba1994b99b99706def1aa5c07 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 20 Aug 2024 13:14:19 -0700 Subject: [PATCH 76/84] non-raw URL --- rapids_config.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rapids_config.cmake b/rapids_config.cmake index bfbaaeebd1c..60aef012ff3 100644 --- a/rapids_config.cmake +++ b/rapids_config.cmake @@ -30,7 +30,7 @@ include(FetchContent) # GIT_TAG = last commit prior to https://github.com/rapidsai/rapids-cmake/pull/678 FetchContent_Declare( rapids-cmake - GIT_REPOSITORY https://raw.githubusercontent.com/rapidsai/rapids-cmake.git + GIT_REPOSITORY https://github.com/rapidsai/rapids-cmake.git GIT_TAG 0e65a8481b9895b5075d2b601690540a21b45deb ) file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-24.10/RAPIDS.cmake From bc701f38bd17837882435695dd83b228a41cf9d0 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 20 Aug 2024 14:25:13 -0700 Subject: [PATCH 77/84] revert rapids-cmake pin --- rapids_config.cmake | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/rapids_config.cmake b/rapids_config.cmake index 60aef012ff3..3a88769f6e7 100644 --- a/rapids_config.cmake +++ b/rapids_config.cmake @@ -26,14 +26,11 @@ else() ) endif() -include(FetchContent) -# GIT_TAG = last commit prior to https://github.com/rapidsai/rapids-cmake/pull/678 -FetchContent_Declare( - rapids-cmake - GIT_REPOSITORY https://github.com/rapidsai/rapids-cmake.git - GIT_TAG 0e65a8481b9895b5075d2b601690540a21b45deb -) -file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-24.10/RAPIDS.cmake - ${CMAKE_CURRENT_BINARY_DIR}/CUDF_RAPIDS-${RAPIDS_VERSION_MAJOR_MINOR}.cmake) - +if(NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/CUDF_RAPIDS-${RAPIDS_VERSION_MAJOR_MINOR}.cmake") + file( + DOWNLOAD + "https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-${RAPIDS_VERSION_MAJOR_MINOR}/RAPIDS.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/CUDF_RAPIDS-${RAPIDS_VERSION_MAJOR_MINOR}.cmake" + ) +endif() include("${CMAKE_CURRENT_BINARY_DIR}/CUDF_RAPIDS-${RAPIDS_VERSION_MAJOR_MINOR}.cmake") From bf59d2d42e59eeb75d4678eab4d0243ce145fed7 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 20 Aug 2024 16:51:59 -0700 Subject: [PATCH 78/84] cudf and pylibcudf need librmm build dependencies for nvtx3-config.cmake --- dependencies.yaml | 2 ++ python/cudf/pyproject.toml | 1 + python/pylibcudf/pyproject.toml | 1 + 3 files changed, 4 insertions(+) diff --git a/dependencies.yaml b/dependencies.yaml index 94c6c368ce5..f5eb6580e17 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -97,6 +97,7 @@ files: - build_python_common - depends_on_pylibcudf - depends_on_libcudf + - depends_on_librmm - depends_on_rmm py_run_cudf: output: pyproject @@ -162,6 +163,7 @@ files: - build_base - build_python_common - depends_on_libcudf + - depends_on_librmm - depends_on_rmm py_run_pylibcudf: output: pyproject diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 5137081312b..cb9fa30afab 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -128,6 +128,7 @@ requires = [ "cmake>=3.26.4,!=3.30.0", "cython>=3.0.3", "libcudf==24.10.*,>=0.0.0a0", + "librmm==24.10.*,>=0.0.0a0", "ninja", "numpy==1.23.*", "pyarrow==16.1.0.*", diff --git a/python/pylibcudf/pyproject.toml b/python/pylibcudf/pyproject.toml index 01c84e3fba2..63d76e9fd4e 100644 --- a/python/pylibcudf/pyproject.toml +++ b/python/pylibcudf/pyproject.toml @@ -103,6 +103,7 @@ requires = [ "cmake>=3.26.4,!=3.30.0", "cython>=3.0.3", "libcudf==24.10.*,>=0.0.0a0", + "librmm==24.10.*,>=0.0.0a0", "ninja", "numpy==1.23.*", "pyarrow==16.1.0.*", From bbbe243e7e8a89878b7d539afdcfe900f68e1f92 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 21 Aug 2024 08:14:25 -0700 Subject: [PATCH 79/84] fix script name --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index d1c6f7f8ac4..c72106d4b77 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -143,7 +143,7 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: pull-request - script: "ci/build_wheel_python.sh" + script: "ci/build_wheel_cudf.sh" wheel-tests-cudf: needs: wheel-build-cudf secrets: inherit From 53ea69b6a73546786a636f0e6f3381a10187897a Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 21 Aug 2024 17:01:25 -0500 Subject: [PATCH 80/84] Apply suggestions from code review Co-authored-by: Bradley Dice --- .github/workflows/build.yaml | 2 +- .github/workflows/pr.yaml | 4 ++-- build.sh | 2 +- ci/build_wheel_cudf.sh | 4 ++-- ci/build_wheel_pylibcudf.sh | 2 +- ci/check_style.sh | 2 -- ci/cudf_pandas_scripts/pandas-tests/run.sh | 4 ++-- ci/release/update-version.sh | 1 + 8 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 263a841fb37..0ea4d5c54dc 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -107,7 +107,7 @@ jobs: date: ${{ inputs.date }} package-name: pylibcudf wheel-build-cudf: - needs: [wheel-publish-libcudf, wheel-publish-pylibcudf] + needs: wheel-publish-pylibcudf secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c72106d4b77..2e2a8b6b9bc 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -138,7 +138,7 @@ jobs: build_type: pull-request script: "ci/build_wheel_pylibcudf.sh" wheel-build-cudf: - needs: [checks, wheel-build-libcudf, wheel-build-pylibcudf] + needs: wheel-build-pylibcudf secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: @@ -152,7 +152,7 @@ jobs: build_type: pull-request script: ci/test_wheel_cudf.sh wheel-build-cudf-polars: - needs: [wheel-build-pylibcudf, wheel-build-cudf] + needs: wheel-build-pylibcudf secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: diff --git a/build.sh b/build.sh index 18fe5569aac..abb64399d52 100755 --- a/build.sh +++ b/build.sh @@ -24,7 +24,7 @@ HELP="$0 [clean] [libcudf] [libcudfwheel] [pylibcudf] [cudf] [cudfjar] [dask_cud libcudf - build the cudf C++ code only pylibcudf - build the pylibcudf Python package cudf - build the cudf Python package - libcudfwheel - build the cudf C++ code packaged as a python wheel package + libcudfwheel - build the cudf C++ code packaged as a Python wheel package cudfjar - build cudf JAR with static libcudf using devtoolset toolchain dask_cudf - build the dask_cudf Python package benchmarks - build benchmarks diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_cudf.sh index 0a4c2e613c6..cf33703f544 100755 --- a/ci/build_wheel_cudf.sh +++ b/ci/build_wheel_cudf.sh @@ -13,9 +13,9 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" # Using env variable PIP_CONSTRAINT is necessary to ensure the constraints # are used when creating the isolated build environment. RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist -RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python /tmp/libcudf_dist +RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python /tmp/pylibcudf_dist echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/libcudf_*.whl)" > /tmp/constraints.txt -echo "pylibcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/pylibcudf_*.whl)" >> /tmp/constraints.txt +echo "pylibcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/pylibcudf_dist/pylibcudf_*.whl)" >> /tmp/constraints.txt export PIP_CONSTRAINT="/tmp/constraints.txt" ./ci/build_wheel.sh ${package_dir} diff --git a/ci/build_wheel_pylibcudf.sh b/ci/build_wheel_pylibcudf.sh index d0468455027..7181a49d397 100755 --- a/ci/build_wheel_pylibcudf.sh +++ b/ci/build_wheel_pylibcudf.sh @@ -8,7 +8,7 @@ package_dir="python/pylibcudf" RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" # Downloads libcudf wheel from this current build, -# then ensures 'cudf' wheel builds always use the 'libcudf' just built in the same CI run. +# then ensures 'pylibcudf' wheel builds always use the 'libcudf' just built in the same CI run. # # Using env variable PIP_CONSTRAINT is necessary to ensure the constraints # are used when creating the isolated build environment. diff --git a/ci/check_style.sh b/ci/check_style.sh index b7b8164cf4f..634d8b0d702 100755 --- a/ci/check_style.sh +++ b/ci/check_style.sh @@ -1,8 +1,6 @@ #!/bin/bash # Copyright (c) 2020-2024, NVIDIA CORPORATION. -exit 0 - set -euo pipefail rapids-logger "Create checks conda environment" diff --git a/ci/cudf_pandas_scripts/pandas-tests/run.sh b/ci/cudf_pandas_scripts/pandas-tests/run.sh index 9a689f3f3fe..e5cd4436a3a 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/run.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/run.sh @@ -13,9 +13,9 @@ rapids-logger "PR number: ${RAPIDS_REF_NAME:-"unknown"}" RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" # Download the cudf, libcudf, and pylibcudf built in the previous step -RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist -RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install \ diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 86737ffa7ab..3a2b10ee8e2 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -52,6 +52,7 @@ DEPENDENCIES=( libcudf libkvikio librmm + pylibcudf rapids-dask-dependency rmm ) From 5916b9416383000b554ae27fd05d75ca0d71ef6b Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 21 Aug 2024 15:03:21 -0700 Subject: [PATCH 81/84] make wheel type explicit in all rapids-download-wheels-from-s3 calls --- ci/cudf_pandas_scripts/run_tests.sh | 4 ++-- ci/test_wheel_cudf.sh | 4 ++-- ci/test_wheel_cudf_polars.sh | 4 ++-- ci/test_wheel_dask_cudf.sh | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index 834045bf5ed..b222f8001e9 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -38,9 +38,9 @@ else RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" # Download the cudf, libcudf, and pylibcudf built in the previous step - RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist + RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist - RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist + RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install \ diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh index 8e340d800ce..6861d699695 100755 --- a/ci/test_wheel_cudf.sh +++ b/ci/test_wheel_cudf.sh @@ -6,9 +6,9 @@ set -eou pipefail RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" # Download the cudf, libcudf, and pylibcudf built in the previous step -RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist -RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install \ diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index c8f0affc7ac..0baf6c9e277 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -18,11 +18,11 @@ else fi RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 python ./dist # Download libcudf and pylibcudf built in the previous step RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist -RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist rapids-logger "Installing cudf_polars and its dependencies" diff --git a/ci/test_wheel_dask_cudf.sh b/ci/test_wheel_dask_cudf.sh index c04769b46b8..fa74b2398f7 100755 --- a/ci/test_wheel_dask_cudf.sh +++ b/ci/test_wheel_dask_cudf.sh @@ -4,12 +4,12 @@ set -eou pipefail RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 python ./dist # Download the cudf, libcudf, and pylibcudf built in the previous step -RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist -RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install \ From 0ebb1679950be42b25da71f21733c7d96f96373b Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 22 Aug 2024 11:33:13 -0500 Subject: [PATCH 82/84] Apply suggestions from code review Co-authored-by: Bradley Dice --- dependencies.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index f5eb6580e17..553d01735b2 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -804,7 +804,7 @@ dependencies: - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file - # This index is needed for rmm, cubinlinker, ptxcompiler. + # This index is needed for libcudf-cu{11,12}. - --extra-index-url=https://pypi.nvidia.com - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple specific: @@ -919,7 +919,7 @@ dependencies: - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file - # This index is needed for rmm-cu{11,12}. + # This index is needed for librmm-cu{11,12}. - --extra-index-url=https://pypi.nvidia.com - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple specific: From 4ba53d84bcfa337baae4baff3a5eb371ea1027b7 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 22 Aug 2024 10:01:47 -0700 Subject: [PATCH 83/84] remove TODO comment about what happens if a system installation of nvcomp is found at build time --- python/libcudf/CMakeLists.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/python/libcudf/CMakeLists.txt b/python/libcudf/CMakeLists.txt index 05bca21ec57..09c7ed2e217 100644 --- a/python/libcudf/CMakeLists.txt +++ b/python/libcudf/CMakeLists.txt @@ -51,9 +51,6 @@ include(../pylibcudf/cmake/Modules/LinkPyarrowHeaders.cmake) add_subdirectory(../../cpp cudf-cpp) # Ensure other libraries needed by libcudf.so get installed alongside it. -# -# TODO: This install is currently overzealous. We should only install the libraries that are -# downloaded by CPM during the build, not libraries that were found on the system. include(cmake/Modules/WheelHelpers.cmake) install_aliased_imported_targets( TARGETS cudf arrow_shared nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp From 5a39dfa33bb05f899ed6d53487a1b8fe403846fb Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 22 Aug 2024 14:12:19 -0700 Subject: [PATCH 84/84] revert build.sh changes --- build.sh | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/build.sh b/build.sh index abb64399d52..957f41aedac 100755 --- a/build.sh +++ b/build.sh @@ -17,14 +17,13 @@ ARGS=$* # script, and that this script resides in the repo dir! REPODIR=$(cd $(dirname $0); pwd) -VALIDARGS="clean libcudf libcudfwheel pylibcudf cudf cudfjar dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz -v -g -n --pydevelop -l --allgpuarch --disable_nvtx --opensource_nvcomp --show_depr_warn --ptds -h --build_metrics --incl_cache_stats --disable_large_strings" -HELP="$0 [clean] [libcudf] [libcudfwheel] [pylibcudf] [cudf] [cudfjar] [dask_cudf] [benchmarks] [tests] [libcudf_kafka] [cudf_kafka] [custreamz] [-v] [-g] [-n] [-h] [--cmake-args=\\\"\\\"] +VALIDARGS="clean libcudf pylibcudf cudf cudfjar dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz -v -g -n --pydevelop -l --allgpuarch --disable_nvtx --opensource_nvcomp --show_depr_warn --ptds -h --build_metrics --incl_cache_stats --disable_large_strings" +HELP="$0 [clean] [libcudf] [pylibcudf] [cudf] [cudfjar] [dask_cudf] [benchmarks] [tests] [libcudf_kafka] [cudf_kafka] [custreamz] [-v] [-g] [-n] [-h] [--cmake-args=\\\"\\\"] clean - remove all existing build artifacts and configuration (start over) libcudf - build the cudf C++ code only pylibcudf - build the pylibcudf Python package cudf - build the cudf Python package - libcudfwheel - build the cudf C++ code packaged as a Python wheel package cudfjar - build cudf JAR with static libcudf using devtoolset toolchain dask_cudf - build the dask_cudf Python package benchmarks - build benchmarks @@ -342,14 +341,6 @@ if buildAll || hasArg libcudf; then fi fi -# Build and install the libcudf Python package -if buildAll || hasArg libcudfwheel; then - - cd ${REPODIR}/python/libcudf - SKBUILD_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${INSTALL_PREFIX};-DCMAKE_LIBRARY_PATH=${LIBCUDF_BUILD_DIR};-DCMAKE_CUDA_ARCHITECTURES=${CUDF_CMAKE_CUDA_ARCHITECTURES};${EXTRA_CMAKE_ARGS}" \ - python ${PYTHON_ARGS_FOR_INSTALL} . -fi - # Build and install the pylibcudf Python package if buildAll || hasArg pylibcudf; then