diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 4ecbbab45..1f3fd7922 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -85,6 +85,7 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: cuspatial + package-type: python wheel-build-cuproj: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 @@ -104,3 +105,4 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: cuproj + package-type: python diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 5e6eb944e..14802741d 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -5,6 +5,7 @@ set -euo pipefail package_name=$1 package_dir=$2 +package_type=$3 source rapids-configure-sccache source rapids-date-string @@ -18,6 +19,9 @@ cd "${package_dir}" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check mkdir -p final_dist -python -m auditwheel repair -w final_dist dist/* +python -m auditwheel repair \ + --exclude libcudf.so \ + -w final_dist \ + dist/* -RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 "${package_type}" final_dist diff --git a/ci/build_wheel_cuproj.sh b/ci/build_wheel_cuproj.sh index 961f4e363..6001e44e9 100755 --- a/ci/build_wheel_cuproj.sh +++ b/ci/build_wheel_cuproj.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. set -euo pipefail -ci/build_wheel.sh cuproj python/cuproj +ci/build_wheel.sh cuproj python/cuproj python diff --git a/ci/build_wheel_cuspatial.sh b/ci/build_wheel_cuspatial.sh index 932665e27..171bfa6a4 100755 --- a/ci/build_wheel_cuspatial.sh +++ b/ci/build_wheel_cuspatial.sh @@ -1,8 +1,6 @@ #!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. set -euo pipefail -export SKBUILD_CMAKE_ARGS="-DUSE_LIBARROW_FROM_PYARROW=ON" - -ci/build_wheel.sh cuspatial python/cuspatial +ci/build_wheel.sh cuspatial python/cuspatial python diff --git a/ci/test_wheel_cuproj.sh b/ci/test_wheel_cuproj.sh index 9451190e0..4144e8dab 100755 --- a/ci/test_wheel_cuproj.sh +++ b/ci/test_wheel_cuproj.sh @@ -5,19 +5,21 @@ set -eou pipefail mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="cuproj_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -# Install additional dependencies +# install build dependencies for fiona apt update DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends libgdal-dev -python -m pip install --no-binary fiona 'fiona>=1.8.19,<1.9' -# Download the cuspatial built in the previous step -RAPIDS_PY_WHEEL_NAME="cuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cuspatial-dep -python -m pip install --no-deps ./local-cuspatial-dep/cuspatial*.whl +# Download the cuproj and cuspatial built in the previous step +RAPIDS_PY_WHEEL_NAME="cuproj_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist +RAPIDS_PY_WHEEL_NAME="cuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist # echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install $(echo ./dist/cuproj*.whl)[test] +python -m pip install \ + --no-binary 'fiona' \ + "$(echo ./dist/cuspatial*.whl)" \ + "$(echo ./dist/cuproj*.whl)[test]" \ + 'fiona>=1.8.19,<1.9' rapids-logger "pytest cuproj" pushd python/cuproj/cuproj diff --git a/ci/test_wheel_cuspatial.sh b/ci/test_wheel_cuspatial.sh index aa36681bc..8faa4e8d3 100755 --- a/ci/test_wheel_cuspatial.sh +++ b/ci/test_wheel_cuspatial.sh @@ -5,15 +5,19 @@ set -eou pipefail mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="cuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -# Install additional dependencies +# install build dependencies for fiona apt update DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends libgdal-dev -python -m pip install --no-binary fiona 'fiona>=1.8.19,<1.9' + +# Download the cuspatial built in the previous step +RAPIDS_PY_WHEEL_NAME="cuspatial_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist # echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install $(echo ./dist/cuspatial*.whl)[test] +python -m pip install \ + --no-binary 'fiona' \ + "$(echo ./dist/cuspatial*.whl)[test]" \ + 'fiona>=1.8.19,<1.9' rapids-logger "pytest cuspatial" pushd python/cuspatial/cuspatial diff --git a/dependencies.yaml b/dependencies.yaml index 3b4bee329..80822b43d 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -17,6 +17,8 @@ files: - depends_on_cudf - depends_on_cuml - depends_on_cupy + - depends_on_libcudf + - depends_on_librmm - rapids_build_skbuild - run_python_cuspatial - test_libcuspatial @@ -77,14 +79,17 @@ files: - build_cpp - build_python - build_wheels + - depends_on_libcudf + - depends_on_librmm py_run_cuspatial: output: [pyproject] pyproject_dir: python/cuspatial extras: table: project includes: - - depends_on_rmm - depends_on_cudf + - depends_on_libcudf + - depends_on_rmm - run_python_cuspatial py_test_cuspatial: output: [pyproject] @@ -141,14 +146,12 @@ dependencies: common: - output_types: [conda, requirements, pyproject] packages: - - ninja - - cmake>=3.26.4,!=3.30.0 + - &ninja ninja + - &cmake cmake>=3.26.4,!=3.30.0 - output_types: conda packages: - c-compiler - cxx-compiler - - libcudf==24.10.*,>=0.0.0a0 - - librmm==24.10.*,>=0.0.0a0 - proj - sqlite specific: @@ -184,13 +187,13 @@ dependencies: common: - output_types: [conda, requirements, pyproject] packages: - - ninja - - cmake>=3.26.4,!=3.30.0 + - *ninja + - *cmake - output_types: conda packages: - c-compiler - cxx-compiler - - librmm==24.10.*,>=0.0.0a0 + - &librmm_unsuffixed librmm==24.10.*,>=0.0.0a0 - proj - sqlite specific: @@ -443,6 +446,31 @@ dependencies: - pylibcudf-cu11==24.10.*,>=0.0.0a0 - {matrix: null, packages: [*cudf_unsuffixed]} + depends_on_libcudf: + common: + - output_types: conda + packages: + - &libcudf_unsuffixed libcudf==24.10.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - libcudf-cu12==24.10.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - libcudf-cu11==24.10.*,>=0.0.0a0 + - {matrix: null, packages: [*libcudf_unsuffixed]} + depends_on_cuml: common: - output_types: conda @@ -523,3 +551,28 @@ dependencies: - libcuspatial==24.10.*,>=0.0.0a0 - cuspatial==24.10.*,>=0.0.0a0 - cuproj==24.10.*,>=0.0.0a0 + + depends_on_librmm: + common: + - output_types: conda + packages: + - *librmm_unsuffixed + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - librmm-cu12==24.10.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - librmm-cu11==24.10.*,>=0.0.0a0 + - {matrix: null, packages: [*librmm_unsuffixed]} diff --git a/python/cuspatial/CMakeLists.txt b/python/cuspatial/CMakeLists.txt index e3678c311..28fcf467d 100644 --- a/python/cuspatial/CMakeLists.txt +++ b/python/cuspatial/CMakeLists.txt @@ -43,15 +43,7 @@ if(NOT cuspatial_FOUND) add_subdirectory(../../cpp cuspatial-cpp EXCLUDE_FROM_ALL) set(cython_lib_dir cuspatial) - include(cmake/Modules/WheelHelpers.cmake) - # TODO: This install is currently overzealous. We should only install the libraries that are - # downloaded by CPM during the build, not libraries that were found on the system. However, in - # practice this would only be a problem if libcudf was not found but some of the - # dependencies were, and we have no real use cases where that happens. - install_aliased_imported_targets( - TARGETS cuspatial arrow_shared nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp - DESTINATION ${cython_lib_dir} - ) + install(TARGETS cuspatial DESTINATION ${cython_lib_dir}) endif() include(rapids-cython-core) diff --git a/python/cuspatial/cmake/Modules/WheelHelpers.cmake b/python/cuspatial/cmake/Modules/WheelHelpers.cmake deleted file mode 100644 index 278d6751c..000000000 --- a/python/cuspatial/cmake/Modules/WheelHelpers.cmake +++ /dev/null @@ -1,59 +0,0 @@ -# ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= -include_guard(GLOBAL) - -# Making libraries available inside wheels by installing the associated targets. -function(install_aliased_imported_targets) - list(APPEND CMAKE_MESSAGE_CONTEXT "install_aliased_imported_targets") - - set(options "") - set(one_value "DESTINATION") - set(multi_value "TARGETS") - cmake_parse_arguments(_ "${options}" "${one_value}" "${multi_value}" ${ARGN}) - - message(VERBOSE "Installing targets '${__TARGETS}' into lib_dir '${__DESTINATION}'") - - foreach(target IN LISTS __TARGETS) - - if(NOT TARGET ${target}) - message(VERBOSE "No target named ${target}") - continue() - endif() - - get_target_property(alias_target ${target} ALIASED_TARGET) - if(alias_target) - set(target ${alias_target}) - endif() - - get_target_property(is_imported ${target} IMPORTED) - if(NOT is_imported) - # If the target isn't imported, install it into the wheel - install(TARGETS ${target} DESTINATION ${__DESTINATION}) - message(VERBOSE "install(TARGETS ${target} DESTINATION ${__DESTINATION})") - else() - # If the target is imported, make sure it's global - get_target_property(type ${target} TYPE) - if(${type} STREQUAL "UNKNOWN_LIBRARY") - install(FILES $ DESTINATION ${__DESTINATION}) - message(VERBOSE "install(FILES $ DESTINATION ${__DESTINATION})") - else() - install(IMPORTED_RUNTIME_ARTIFACTS ${target} DESTINATION ${__DESTINATION}) - message( - VERBOSE - "install(IMPORTED_RUNTIME_ARTIFACTS $ DESTINATION ${__DESTINATION})" - ) - endif() - endif() - endforeach() -endfunction() diff --git a/python/cuspatial/cuspatial/__init__.py b/python/cuspatial/cuspatial/__init__.py index 069b5454d..14a5fcc4d 100644 --- a/python/cuspatial/cuspatial/__init__.py +++ b/python/cuspatial/cuspatial/__init__.py @@ -1,4 +1,8 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +# load cudf before any other imports, to be sure libcudf's symbols are found +# in the libcudf.so from the libcudf wheel (if one is installed) +import cudf from ._version import __git_commit__, __version__ from .core.geodataframe import GeoDataFrame diff --git a/python/cuspatial/pyproject.toml b/python/cuspatial/pyproject.toml index 80f436ea6..25ea28bfd 100644 --- a/python/cuspatial/pyproject.toml +++ b/python/cuspatial/pyproject.toml @@ -32,6 +32,7 @@ requires-python = ">=3.10" dependencies = [ "cudf==24.10.*,>=0.0.0a0", "geopandas>=0.11.0", + "libcudf==24.10.*,>=0.0.0a0", "numpy>=1.23,<2.0a0", "rmm==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. @@ -71,6 +72,7 @@ known_dask = [ known_rapids = [ "rmm", "cudf", + "libcudf", "pylibcudf", ] known_first_party = [ @@ -134,6 +136,8 @@ requires = [ "cmake>=3.26.4,!=3.30.0", "cudf==24.10.*,>=0.0.0a0", "cython>=3.0.0", + "libcudf==24.10.*,>=0.0.0a0", + "librmm==24.10.*,>=0.0.0a0", "ninja", "rmm==24.10.*,>=0.0.0a0", "wheel",