From 9ccf8654bf62a4c5609d26e688ab7039cbdda803 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 9 Jul 2024 16:27:21 -0400 Subject: [PATCH 01/14] Use dynamic CUDA libraries --- ci/build_wheel.sh | 24 ++++++++++++++++++++++-- dependencies.yaml | 19 +++++++++++++++++++ python/pylibraft/CMakeLists.txt | 23 ++++++++++++++++++++++- python/pylibraft/pyproject.toml | 4 ++++ 4 files changed, 67 insertions(+), 3 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 62d93a668e..27f6cb2c80 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -20,10 +20,30 @@ rapids-generate-version > VERSION cd "${package_dir}" +case "${RAPIDS_CUDA_VERSION}" in + 12.*) + EXCLUDE_ARGS=( + --exclude "libcublas.so.12" + --exclude "libcublasLt.so.12" + --exclude "libcurand.so.10" + --exclude "libcusolver.so.11" + --exclude "libcusparse.so.12" + --exclude "libnvJitLink.so.12" + --exclude "libucp.so.0" + ) + EXTRA_CMAKE_ARGS="-DUSE_CUDA_MATH_WHEELS=ON" + ;; + 11.*) + EXCLUDE_ARGS=() + EXTRA_CMAKE_ARGS="-DUSE_CUDA_MATH_WHEELS=OFF" + ;; +esac + # Hardcode the output dir -python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check +SKBUILD_CMAKE_ARGS="${EXTRA_CMAKE_ARGS}" \ + python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check mkdir -p final_dist -python -m auditwheel repair -w final_dist --exclude "libucp.so.0" dist/* +python -m auditwheel repair -w final_dist "${EXCLUDE_ARGS[@]}" dist/* RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist diff --git a/dependencies.yaml b/dependencies.yaml index 630b5a50c9..3e7164e470 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -83,6 +83,7 @@ files: extras: table: project includes: + - cuda - run_pylibraft py_test_pylibraft: output: pyproject @@ -377,6 +378,24 @@ dependencies: - *libcusolver114 - *libcusparse_dev114 - *libcusparse114 + - output_types: pyproject + matrices: + - matrix: + cuda: "12.*" + packages: + - nvidia-cublas-cu12 + - nvidia-curand-cu12 + - nvidia-cusolver-cu12 + - nvidia-cusparse-cu12 + - matrix: + cuda: "11.*" + packages: + - matrix: + packages: + - nvidia-cublas + - nvidia-curand + - nvidia-cusolver + - nvidia-cusparse depends_on_cupy: common: diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index 6cbe8e4cbf..4a85318d3c 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -30,6 +30,7 @@ project( option(FIND_RAFT_CPP "Search for existing RAFT C++ installations before defaulting to local files" ON ) +option(USE_CUDA_MATH_WHEELS "Use the CUDA math wheels instead of the system libraries" OFF) # If the user requested it we attempt to find RAFT. if(FIND_RAFT_CPP) @@ -48,15 +49,35 @@ endif() include(rapids-cython-core) if(NOT raft_FOUND) + find_package(CUDAToolkit REQUIRED) + set(BUILD_TESTS OFF) set(BUILD_PRIMS_BENCH OFF) set(BUILD_ANN_BENCH OFF) set(RAFT_COMPILE_LIBRARY ON) set(CUDA_STATIC_RUNTIME ON) - set(CUDA_STATIC_MATH_LIBRARIES ON) + if(CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12.0) + set(CUDA_STATIC_MATH_LIBRARIES OFF) + else() + if(USE_CUDA_MATH_WHEELS) + message(FATAL_ERROR "CAnnot use CUDA math wheels with CUDA < 12.0") + endif() + set(CUDA_STATIC_MATH_LIBRARIES ON) + endif() add_subdirectory(../../cpp raft-cpp EXCLUDE_FROM_ALL) + if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) + set_property(TARGET raft PROPERTY INSTALL_RPATH + "$ORIGIN/../nvidia/cublas/lib" + "$ORIGIN/../nvidia/cufft/lib" + "$ORIGIN/../nvidia/curand/lib" + "$ORIGIN/../nvidia/cusolver/lib" + "$ORIGIN/../nvidia/cusparse/lib" + "$ORIGIN/../nvidia/nvjitlink/lib" + ) + endif() + # When building the C++ libraries from source we must copy libraft.so alongside the # pairwise_distance and random Cython libraries TODO: when we have a single 'compiled' raft # library, we shouldn't need this diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index 4c1d8e8fa1..a65203c615 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -33,6 +33,10 @@ requires-python = ">=3.9" dependencies = [ "cuda-python", "numpy>=1.23,<2.0a0", + "nvidia-cublas", + "nvidia-curand", + "nvidia-cusolver", + "nvidia-cusparse", "rmm==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ From 92ee07c21aee1f730addc0fbfe155908cf88cee1 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Mon, 19 Aug 2024 15:34:46 -0400 Subject: [PATCH 02/14] Unconditionall exclude libucp --- ci/build_wheel.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 27f6cb2c80..63538177b6 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -34,7 +34,9 @@ case "${RAPIDS_CUDA_VERSION}" in EXTRA_CMAKE_ARGS="-DUSE_CUDA_MATH_WHEELS=ON" ;; 11.*) - EXCLUDE_ARGS=() + EXCLUDE_ARGS=( + --exclude "libucp.so.0" + ) EXTRA_CMAKE_ARGS="-DUSE_CUDA_MATH_WHEELS=OFF" ;; esac From d6b37140f1e3f48f029c82b0b6f4b6e24e98d75d Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Mon, 19 Aug 2024 15:44:55 -0400 Subject: [PATCH 03/14] Fix SKBUILD_CMAKE_ARGS --- ci/build_wheel.sh | 5 +---- ci/build_wheel_pylibraft.sh | 10 +++++++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 63538177b6..e7ae52f33a 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -31,19 +31,16 @@ case "${RAPIDS_CUDA_VERSION}" in --exclude "libnvJitLink.so.12" --exclude "libucp.so.0" ) - EXTRA_CMAKE_ARGS="-DUSE_CUDA_MATH_WHEELS=ON" ;; 11.*) EXCLUDE_ARGS=( --exclude "libucp.so.0" ) - EXTRA_CMAKE_ARGS="-DUSE_CUDA_MATH_WHEELS=OFF" ;; esac # Hardcode the output dir -SKBUILD_CMAKE_ARGS="${EXTRA_CMAKE_ARGS}" \ - python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check +python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check mkdir -p final_dist python -m auditwheel repair -w final_dist "${EXCLUDE_ARGS[@]}" dist/* diff --git a/ci/build_wheel_pylibraft.sh b/ci/build_wheel_pylibraft.sh index 895c311f46..a094798a65 100755 --- a/ci/build_wheel_pylibraft.sh +++ b/ci/build_wheel_pylibraft.sh @@ -3,7 +3,15 @@ set -euo pipefail +case "${RAPIDS_CUDA_VERSION}" in + 12.*) + EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=ON" + 11.*) + EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=OFF" + ;; +esac + # Set up skbuild options. Enable sccache in skbuild config options -export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF" +export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF${EXTRA_CMAKE_ARGS}" ci/build_wheel.sh pylibraft python/pylibraft From 5bcd36f57652f082e7c995590f27636ab67bc8a6 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Mon, 19 Aug 2024 15:46:25 -0400 Subject: [PATCH 04/14] Review feedback --- python/pylibraft/CMakeLists.txt | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index 4a85318d3c..3fefa436af 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -56,13 +56,11 @@ if(NOT raft_FOUND) set(BUILD_ANN_BENCH OFF) set(RAFT_COMPILE_LIBRARY ON) set(CUDA_STATIC_RUNTIME ON) + set(CUDA_STATIC_MATH_LIBRARIES ON) if(CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12.0) set(CUDA_STATIC_MATH_LIBRARIES OFF) - else() - if(USE_CUDA_MATH_WHEELS) - message(FATAL_ERROR "CAnnot use CUDA math wheels with CUDA < 12.0") - endif() - set(CUDA_STATIC_MATH_LIBRARIES ON) + elseif(USE_CUDA_MATH_WHEELS) + message(FATAL_ERROR "CAnnot use CUDA math wheels with CUDA < 12.0") endif() add_subdirectory(../../cpp raft-cpp EXCLUDE_FROM_ALL) From f2d4e98538d8073f4020da47538e533032c4ee59 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Mon, 19 Aug 2024 15:46:43 -0400 Subject: [PATCH 05/14] Oops --- python/pylibraft/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index 3fefa436af..df89cdfc6a 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -60,7 +60,7 @@ if(NOT raft_FOUND) if(CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12.0) set(CUDA_STATIC_MATH_LIBRARIES OFF) elseif(USE_CUDA_MATH_WHEELS) - message(FATAL_ERROR "CAnnot use CUDA math wheels with CUDA < 12.0") + message(FATAL_ERROR "Cannot use CUDA math wheels with CUDA < 12.0") endif() add_subdirectory(../../cpp raft-cpp EXCLUDE_FROM_ALL) From e2073ff798d5761b4a781916b4e98ef0ec135ce7 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Mon, 19 Aug 2024 15:51:56 -0400 Subject: [PATCH 06/14] Syntax --- ci/build_wheel_pylibraft.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/build_wheel_pylibraft.sh b/ci/build_wheel_pylibraft.sh index a094798a65..ce9f0ed172 100755 --- a/ci/build_wheel_pylibraft.sh +++ b/ci/build_wheel_pylibraft.sh @@ -6,6 +6,7 @@ set -euo pipefail case "${RAPIDS_CUDA_VERSION}" in 12.*) EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=ON" + ;; 11.*) EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=OFF" ;; From d4e050f10dcd4c054e32d4e3764d7be6c64e6a97 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Mon, 19 Aug 2024 17:31:29 -0400 Subject: [PATCH 07/14] Set rpath on correct target --- python/pylibraft/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index df89cdfc6a..6e7b5252a2 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -66,7 +66,7 @@ if(NOT raft_FOUND) add_subdirectory(../../cpp raft-cpp EXCLUDE_FROM_ALL) if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) - set_property(TARGET raft PROPERTY INSTALL_RPATH + set_property(TARGET raft_lib PROPERTY INSTALL_RPATH "$ORIGIN/../nvidia/cublas/lib" "$ORIGIN/../nvidia/cufft/lib" "$ORIGIN/../nvidia/curand/lib" From 4adbc97d88241ba84c2eea8485381e8d62aee673 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 20 Aug 2024 15:21:20 -0400 Subject: [PATCH 08/14] Remove cufft from rpath --- python/pylibraft/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index 6e7b5252a2..c286d3debf 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -68,7 +68,6 @@ if(NOT raft_FOUND) if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) set_property(TARGET raft_lib PROPERTY INSTALL_RPATH "$ORIGIN/../nvidia/cublas/lib" - "$ORIGIN/../nvidia/cufft/lib" "$ORIGIN/../nvidia/curand/lib" "$ORIGIN/../nvidia/cusolver/lib" "$ORIGIN/../nvidia/cusparse/lib" From 6d772a763d8e0dab15d033c6638f04302ab96c15 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Wed, 21 Aug 2024 09:40:28 -0400 Subject: [PATCH 09/14] Add matrix entries --- ci/build_wheel.sh | 2 +- dependencies.yaml | 7 +++++++ python/pylibraft/pyproject.toml | 4 ---- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index e7ae52f33a..772863d614 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -40,7 +40,7 @@ case "${RAPIDS_CUDA_VERSION}" in esac # Hardcode the output dir -python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check +python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check --config-settings=rapidsai.matrix_entry="cuda_suffixed=true;use_cuda_wheels=true" mkdir -p final_dist python -m auditwheel repair -w final_dist "${EXCLUDE_ARGS[@]}" dist/* diff --git a/dependencies.yaml b/dependencies.yaml index 3e7164e470..2b91308e23 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -382,6 +382,8 @@ dependencies: matrices: - matrix: cuda: "12.*" + cuda_suffixed: "true" + use_cuda_wheels: "true" packages: - nvidia-cublas-cu12 - nvidia-curand-cu12 @@ -389,13 +391,18 @@ dependencies: - nvidia-cusparse-cu12 - matrix: cuda: "11.*" + cuda_suffixed: "true" + use_cuda_wheels: "true" packages: - matrix: + use_cuda_wheels: "true" packages: - nvidia-cublas - nvidia-curand - nvidia-cusolver - nvidia-cusparse + - matrix: + packages: depends_on_cupy: common: diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index a65203c615..4c1d8e8fa1 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -33,10 +33,6 @@ requires-python = ">=3.9" dependencies = [ "cuda-python", "numpy>=1.23,<2.0a0", - "nvidia-cublas", - "nvidia-curand", - "nvidia-cusolver", - "nvidia-cusparse", "rmm==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ From d20ae1a59a40300305107e8e1e3a6a6ddc3fe5a5 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Wed, 21 Aug 2024 09:51:30 -0400 Subject: [PATCH 10/14] Make use_cuda_wheels the default --- ci/build_wheel.sh | 2 +- dependencies.yaml | 6 +++--- python/pylibraft/pyproject.toml | 6 +++++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 772863d614..e7ae52f33a 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -40,7 +40,7 @@ case "${RAPIDS_CUDA_VERSION}" in esac # Hardcode the output dir -python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check --config-settings=rapidsai.matrix_entry="cuda_suffixed=true;use_cuda_wheels=true" +python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check mkdir -p final_dist python -m auditwheel repair -w final_dist "${EXCLUDE_ARGS[@]}" dist/* diff --git a/dependencies.yaml b/dependencies.yaml index 2b91308e23..385dfa0290 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -395,14 +395,14 @@ dependencies: use_cuda_wheels: "true" packages: - matrix: - use_cuda_wheels: "true" + use_cuda_wheels: "false" + packages: + - matrix: packages: - nvidia-cublas - nvidia-curand - nvidia-cusolver - nvidia-cusparse - - matrix: - packages: depends_on_cupy: common: diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index 4c1d8e8fa1..9a826e53c6 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -33,6 +33,10 @@ requires-python = ">=3.9" dependencies = [ "cuda-python", "numpy>=1.23,<2.0a0", + "nvidia-cublas", + "nvidia-curand", + "nvidia-cusolver", + "nvidia-cusparse", "rmm==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ @@ -124,7 +128,7 @@ requires = [ "rmm==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. dependencies-file = "../../dependencies.yaml" -matrix-entry = "cuda_suffixed=true" +matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" [tool.pytest.ini_options] filterwarnings = [ From 896a394fbbdd4fe1fb2e21e31e26fef0ae976998 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Wed, 21 Aug 2024 09:56:14 -0400 Subject: [PATCH 11/14] Move cuda wheels into their own dependency set --- dependencies.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dependencies.yaml b/dependencies.yaml index 385dfa0290..d6ea0fe4a6 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -83,7 +83,7 @@ files: extras: table: project includes: - - cuda + - cuda_wheels - run_pylibraft py_test_pylibraft: output: pyproject @@ -378,6 +378,8 @@ dependencies: - *libcusolver114 - *libcusparse_dev114 - *libcusparse114 + cuda_wheels: + specific: - output_types: pyproject matrices: - matrix: From 1e1fe91f7a410d81a59967ec66354dad2c91f784 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Wed, 21 Aug 2024 10:39:32 -0400 Subject: [PATCH 12/14] Remove unneeded cuda_suffixed entry --- dependencies.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index d6ea0fe4a6..26cd3a39fe 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -384,7 +384,6 @@ dependencies: matrices: - matrix: cuda: "12.*" - cuda_suffixed: "true" use_cuda_wheels: "true" packages: - nvidia-cublas-cu12 @@ -393,7 +392,6 @@ dependencies: - nvidia-cusparse-cu12 - matrix: cuda: "11.*" - cuda_suffixed: "true" use_cuda_wheels: "true" packages: - matrix: From c68119af97f556de66e9f73350e59a3d504d6050 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Wed, 21 Aug 2024 10:50:32 -0400 Subject: [PATCH 13/14] Add explanatory comments to dependencies.yaml --- dependencies.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dependencies.yaml b/dependencies.yaml index 26cd3a39fe..3a595630ca 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -394,9 +394,13 @@ dependencies: cuda: "11.*" use_cuda_wheels: "true" packages: + # if use_cuda_wheels=false is provided, do not add dependencies on any CUDA wheels + # (e.g. for DLFW and pip devcontainers) - matrix: use_cuda_wheels: "false" packages: + # if no matching matrix selectors passed, list the unsuffixed packages + # (just as a source of documentation, as this populates pyproject.toml in source control) - matrix: packages: - nvidia-cublas From 7f663a03504c555b7ccd4c9c34f6fcd395a0bef9 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Wed, 21 Aug 2024 17:05:42 -0400 Subject: [PATCH 14/14] Add note about CUDA 11's lack of wheels --- dependencies.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/dependencies.yaml b/dependencies.yaml index 3a595630ca..92c6d98414 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -390,6 +390,7 @@ dependencies: - nvidia-curand-cu12 - nvidia-cusolver-cu12 - nvidia-cusparse-cu12 + # CUDA 11 does not provide wheels, so use the system libraries instead - matrix: cuda: "11.*" use_cuda_wheels: "true"