From e414ab9cdf3c0ec2191212cc54ca8e9e9f9a2d64 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 23 Jul 2024 12:34:40 -0500 Subject: [PATCH] Explicitly select for CUDA-suffixed dependencies in `rapids-make-pip-dependencies` (#365) * select CUDA-suffixed dependencies for pip env * change more places * expand comment * one more place * bump version * move matrix entry to argument * always keep cuda, arch, and py in the rapids-dependency-file-generator --matrix * Update features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh Co-authored-by: Paul Taylor <178183+trxcllnt@users.noreply.github.com> * update make-conda-dependencies too --------- Co-authored-by: Paul Taylor <178183+trxcllnt@users.noreply.github.com> --- .../devcontainer-feature.json | 2 +- .../bin/make-conda-dependencies.sh | 50 +++++++++------ .../bin/make-pip-dependencies.sh | 62 +++++++++++++------ 3 files changed, 77 insertions(+), 37 deletions(-) diff --git a/features/src/rapids-build-utils/devcontainer-feature.json b/features/src/rapids-build-utils/devcontainer-feature.json index a7803af2..08ae9d90 100644 --- a/features/src/rapids-build-utils/devcontainer-feature.json +++ b/features/src/rapids-build-utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "NVIDIA RAPIDS devcontainer build utilities", "id": "rapids-build-utils", - "version": "24.8.14", + "version": "24.8.15", "description": "A feature to install the RAPIDS devcontainer build utilities", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh index 99d44806..9087946b 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh @@ -12,6 +12,9 @@ # -e,--exclude Path(s) to requirement files of packages to exclude. # -i,--include Path(s) to requirement files of packages to include. # -k,--key Only include the key(s) +# --matrix-entry Matrix entries, in the form 'key=value' to be added to the '--matrix' arg +# of rapids-dependency-file-generator. +# (can be passed multiple times) # @_include_value_options rapids-list-repos -h | tail -n+2 | head -n-3; # --repo Only include dependencies for repo(s). # (default: all repositories) @@ -41,6 +44,7 @@ make_conda_dependencies() { test ${#exclude[@]} -eq 0 && exclude=(); test ${#include[@]} -eq 0 && include=(); test ${#key[@]} -eq 0 && key=(all); + test ${#matrix_entry[@]} -eq 0 && matrix_entry=(); local -a _exclude=(); local exc; for exc in "${exclude[@]}"; do @@ -58,6 +62,16 @@ make_conda_dependencies() { local python_version="${PYTHON_VERSION:-$(python3 --version 2>&1 | cut -d' ' -f2)}"; python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")"; + local -a _matrix_selectors=( + arch="$(uname -m)" + cuda="${cuda_version}" + py="${python_version}" + ); + + # add extra arguments (if there are conflicts, e.g. 'py=3.10;py=3.11', it's fine... the last one will win) + test ${#matrix_entry[@]} -gt 0 && _matrix_selectors+=("${matrix_entry[@]}"); + local -r matrix_selectors=$(IFS=";"; echo "${_matrix_selectors[*]}") + local conda_env_yamls=(); local i; @@ -78,12 +92,12 @@ make_conda_dependencies() { for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do local file="/tmp/${!repo_name}.${repo_keys[$keyi]}.env.yaml"; conda_env_yamls+=("${file}"); - generate_env_yaml \ - "${file}" \ - --file-key "${repo_keys[$keyi]}" \ - --output conda \ - --config ~/"${!repo_path}/dependencies.yaml" \ - --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ + generate_env_yaml \ + "${file}" \ + --file-key "${repo_keys[$keyi]}" \ + --output conda \ + --config ~/"${!repo_path}/dependencies.yaml" \ + --matrix "${matrix_selectors}" \ ; done @@ -100,12 +114,12 @@ make_conda_dependencies() { for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do local file="/tmp/${!repo_name}.lib${!cpp_name}.${repo_keys[$keyi]}.env.yaml"; conda_env_yamls+=("${file}"); - generate_env_yaml \ - "${file}" \ - --file-key "${repo_keys[$keyi]}" \ - --output conda \ - --config ~/"${!repo_path}/dependencies.yaml" \ - --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ + generate_env_yaml \ + "${file}" \ + --file-key "${repo_keys[$keyi]}" \ + --output conda \ + --config ~/"${!repo_path}/dependencies.yaml" \ + --matrix "${matrix_selectors}" \ ; done done @@ -123,12 +137,12 @@ make_conda_dependencies() { for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do local file="/tmp/${!repo_name}.${!py_name}.${repo_keys[$keyi]}.env.yaml"; conda_env_yamls+=("${file}"); - generate_env_yaml \ - "${file}" \ - --file-key "${repo_keys[$keyi]}" \ - --output conda \ - --config ~/"${!repo_path}/dependencies.yaml" \ - --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ + generate_env_yaml \ + "${file}" \ + --file-key "${repo_keys[$keyi]}" \ + --output conda \ + --config ~/"${!repo_path}/dependencies.yaml" \ + --matrix "${matrix_selectors}" \ ; done done diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh index 18121243..af4b818d 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh @@ -13,6 +13,9 @@ # -e,--exclude Path(s) to requirement files of packages to exclude. # -i,--include Path(s) to requirement files of packages to include. # -k,--key Only include the key(s) +# --matrix-entry Matrix entries, in the form 'key=value' to be added to the '--matrix' arg +# of rapids-dependency-file-generator. +# (can be passed multiple times) # @_include_value_options rapids-list-repos -h | tail -n+2 | head -n-3; # --repo Only include dependencies for repo(s). # (default: all repositories) @@ -41,6 +44,7 @@ make_pip_dependencies() { test ${#exclude[@]} -eq 0 && exclude=(); test ${#include[@]} -eq 0 && include=(); test ${#key[@]} -eq 0 && key=(py_build py_run py_test all); + test ${#matrix_entry[@]} -eq 0 && matrix_entry=(); test ${#requirement[@]} -eq 0 && requirement=(); local -a _exclude=(); @@ -59,6 +63,28 @@ make_pip_dependencies() { local python_version="${PYTHON_VERSION:-$(python3 --version 2>&1 | cut -d' ' -f2)}"; python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")"; + # Why default to cuda_suffixed=true? + # + # Projects that depend on different pip libraries across different CUDA versions + # (e.g. 'cudf' only depending on 'pynvjitlink' from CUDA 12.0 onwards), split up their + # dependency lists with 'cuda_suffixed={true,false}'. + # + # Here we want the suffixed versions (like 'pynvjitlink-cu12'). + # + # It's ok for other RAPIDS libraries to end up in this list (like 'rmm-cu12')... in builds + # where those are also being built in the devcontainer, they'll be filtered out via + # inclusion in the 'pip_noinstall' list below. + local -a _matrix_selectors=( + arch="$(uname -m)" + cuda="${cuda_version}" + cuda_suffixed=true + py="${python_version}" + ); + + # add extra arguments (if there are conflicts, e.g. 'py=3.10;py=3.11', it's fine... the last one will win) + test ${#matrix_entry[@]} -gt 0 && _matrix_selectors+=("${matrix_entry[@]}"); + local -r matrix_selectors=$(IFS=";"; echo "${_matrix_selectors[*]}") + local pip_reqs_txts=(); eval "$(rapids-list-repos "${OPTS[@]}")"; @@ -81,12 +107,12 @@ make_pip_dependencies() { for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do local file="/tmp/${!repo_name}.${repo_keys[$keyi]}.requirements.txt"; pip_reqs_txts+=("${file}"); - generate_requirements \ - "${file}" \ - --file-key "${repo_keys[$keyi]}" \ - --output requirements \ - --config ~/"${!repo_path}/dependencies.yaml" \ - --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ + generate_requirements \ + "${file}" \ + --file-key "${repo_keys[$keyi]}" \ + --output requirements \ + --config ~/"${!repo_path}/dependencies.yaml" \ + --matrix "${matrix_selectors}" \ ; done @@ -103,12 +129,12 @@ make_pip_dependencies() { for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do local file="/tmp/${!repo_name}.lib${!cpp_name}.${repo_keys[$keyi]}.requirements.txt"; pip_reqs_txts+=("${file}"); - generate_requirements \ - "${file}" \ - --file-key "${repo_keys[$keyi]}" \ - --output requirements \ - --config ~/"${!repo_path}/dependencies.yaml" \ - --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ + generate_requirements \ + "${file}" \ + --file-key "${repo_keys[$keyi]}" \ + --output requirements \ + --config ~/"${!repo_path}/dependencies.yaml" \ + --matrix "${matrix_selectors}" \ ; done done @@ -126,12 +152,12 @@ make_pip_dependencies() { for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do local file="/tmp/${!repo_name}.${!py_name}.${repo_keys[$keyi]}.requirements.txt"; pip_reqs_txts+=("${file}"); - generate_requirements \ - "${file}" \ - --file-key "${repo_keys[$keyi]}" \ - --output requirements \ - --config ~/"${!repo_path}/dependencies.yaml" \ - --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ + generate_requirements \ + "${file}" \ + --file-key "${repo_keys[$keyi]}" \ + --output requirements \ + --config ~/"${!repo_path}/dependencies.yaml" \ + --matrix "${matrix_selectors}" \ ; done done