Explicitly select for CUDA-suffixed dependencies in `rapids-make-pip-…

…dependencies` (#365) * select CUDA-suffixed dependencies for pip env * change more places * expand comment * one more place * bump version * move matrix entry to argument * always keep cuda, arch, and py in the rapids-dependency-file-generator --matrix * Update features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh Co-authored-by: Paul Taylor <178183+trxcllnt@users.noreply.github.com> * update make-conda-dependencies too --------- Co-authored-by: Paul Taylor <178183+trxcllnt@users.noreply.github.com>
rapidsai · Jul 23, 2024 · e414ab9 · e414ab9
1 parent 623a7bf
commit e414ab9
Show file tree

Hide file tree

Showing 3 changed files with 77 additions and 37 deletions.
diff --git a/features/src/rapids-build-utils/devcontainer-feature.json b/features/src/rapids-build-utils/devcontainer-feature.json
@@ -1,7 +1,7 @@
 {
  "name": "NVIDIA RAPIDS devcontainer build utilities",
  "id": "rapids-build-utils",
- "version": "24.8.14",
+ "version": "24.8.15",
  "description": "A feature to install the RAPIDS devcontainer build utilities",
  "containerEnv": {
  "BASH_ENV": "/etc/bash.bash_env"

diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh
@@ -12,6 +12,9 @@
 # -e,--exclude <file> Path(s) to requirement files of packages to exclude.
 # -i,--include <file> Path(s) to requirement files of packages to include.
 # -k,--key <key> Only include the key(s)
+# --matrix-entry <entry> Matrix entries, in the form 'key=value' to be added to the '--matrix' arg
+# of rapids-dependency-file-generator.
+# (can be passed multiple times)
 # @_include_value_options rapids-list-repos -h | tail -n+2 | head -n-3;
 # --repo <repo> Only include dependencies for repo(s).
 # (default: all repositories)
@@ -41,6 +44,7 @@ make_conda_dependencies() {
  test ${#exclude[@]} -eq 0 && exclude=();
  test ${#include[@]} -eq 0 && include=();
  test ${#key[@]} -eq 0 && key=(all);
+ test ${#matrix_entry[@]} -eq 0 && matrix_entry=();
 
  local -a _exclude=();
  local exc; for exc in "${exclude[@]}"; do
@@ -58,6 +62,16 @@ make_conda_dependencies() {
  local python_version="${PYTHON_VERSION:-$(python3 --version 2>&1 | cut -d' ' -f2)}";
  python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")";
 
+ local -a _matrix_selectors=(
+ arch="$(uname -m)"
+ cuda="${cuda_version}"
+ py="${python_version}"
+ );
+
+ # add extra arguments (if there are conflicts, e.g. 'py=3.10;py=3.11', it's fine... the last one will win)
+ test ${#matrix_entry[@]} -gt 0 && _matrix_selectors+=("${matrix_entry[@]}");
+ local -r matrix_selectors=$(IFS=";"; echo "${_matrix_selectors[*]}")
+
  local conda_env_yamls=();
 
  local i;
@@ -78,12 +92,12 @@ make_conda_dependencies() {
  for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do
  local file="/tmp/${!repo_name}.${repo_keys[$keyi]}.env.yaml";
  conda_env_yamls+=("${file}");
- generate_env_yaml  \
- "${file}"  \
- --file-key "${repo_keys[$keyi]}"  \
- --output conda  \
- --config ~/"${!repo_path}/dependencies.yaml"  \
- --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \
+ generate_env_yaml \
+ "${file}" \
+ --file-key "${repo_keys[$keyi]}" \
+ --output conda \
+ --config ~/"${!repo_path}/dependencies.yaml" \
+ --matrix "${matrix_selectors}"  \
  ;
  done
 
@@ -100,12 +114,12 @@ make_conda_dependencies() {
  for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do
  local file="/tmp/${!repo_name}.lib${!cpp_name}.${repo_keys[$keyi]}.env.yaml";
  conda_env_yamls+=("${file}");
- generate_env_yaml  \
- "${file}"  \
- --file-key "${repo_keys[$keyi]}"  \
- --output conda  \
- --config ~/"${!repo_path}/dependencies.yaml"  \
- --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \
+ generate_env_yaml \
+ "${file}" \
+ --file-key "${repo_keys[$keyi]}" \
+ --output conda \
+ --config ~/"${!repo_path}/dependencies.yaml" \
+ --matrix "${matrix_selectors}"  \
  ;
  done
  done
@@ -123,12 +137,12 @@ make_conda_dependencies() {
  for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do
  local file="/tmp/${!repo_name}.${!py_name}.${repo_keys[$keyi]}.env.yaml";
  conda_env_yamls+=("${file}");
- generate_env_yaml  \
- "${file}"  \
- --file-key "${repo_keys[$keyi]}"  \
- --output conda  \
- --config ~/"${!repo_path}/dependencies.yaml"  \
- --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \
+ generate_env_yaml \
+ "${file}" \
+ --file-key "${repo_keys[$keyi]}" \
+ --output conda \
+ --config ~/"${!repo_path}/dependencies.yaml" \
+ --matrix "${matrix_selectors}"  \
  ;
  done
  done

diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh
@@ -13,6 +13,9 @@
 # -e,--exclude <file> Path(s) to requirement files of packages to exclude.
 # -i,--include <file> Path(s) to requirement files of packages to include.
 # -k,--key <key> Only include the key(s)
+# --matrix-entry <entry> Matrix entries, in the form 'key=value' to be added to the '--matrix' arg
+# of rapids-dependency-file-generator.
+# (can be passed multiple times)
 # @_include_value_options rapids-list-repos -h | tail -n+2 | head -n-3;
 # --repo <repo> Only include dependencies for repo(s).
 # (default: all repositories)
@@ -41,6 +44,7 @@ make_pip_dependencies() {
  test ${#exclude[@]} -eq 0 && exclude=();
  test ${#include[@]} -eq 0 && include=();
  test ${#key[@]} -eq 0 && key=(py_build py_run py_test all);
+ test ${#matrix_entry[@]} -eq 0 && matrix_entry=();
  test ${#requirement[@]} -eq 0 && requirement=();
 
  local -a _exclude=();
@@ -59,6 +63,28 @@ make_pip_dependencies() {
  local python_version="${PYTHON_VERSION:-$(python3 --version 2>&1 | cut -d' ' -f2)}";
  python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")";
 
+ # Why default to cuda_suffixed=true?
+ #
+ # Projects that depend on different pip libraries across different CUDA versions
+ # (e.g. 'cudf' only depending on 'pynvjitlink' from CUDA 12.0 onwards), split up their
+ # dependency lists with 'cuda_suffixed={true,false}'.
+ #
+ # Here we want the suffixed versions (like 'pynvjitlink-cu12').
+ #
+ # It's ok for other RAPIDS libraries to end up in this list (like 'rmm-cu12')... in builds
+ # where those are also being built in the devcontainer, they'll be filtered out via
+ # inclusion in the 'pip_noinstall' list below.
+ local -a _matrix_selectors=(
+ arch="$(uname -m)"
+ cuda="${cuda_version}"
+ cuda_suffixed=true
+ py="${python_version}"
+ );
+
+ # add extra arguments (if there are conflicts, e.g. 'py=3.10;py=3.11', it's fine... the last one will win)
+ test ${#matrix_entry[@]} -gt 0 && _matrix_selectors+=("${matrix_entry[@]}");
+ local -r matrix_selectors=$(IFS=";"; echo "${_matrix_selectors[*]}")
+
  local pip_reqs_txts=();
 
  eval "$(rapids-list-repos "${OPTS[@]}")";
@@ -81,12 +107,12 @@ make_pip_dependencies() {
  for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do
  local file="/tmp/${!repo_name}.${repo_keys[$keyi]}.requirements.txt";
  pip_reqs_txts+=("${file}");
- generate_requirements  \
- "${file}"  \
- --file-key "${repo_keys[$keyi]}"  \
- --output requirements  \
- --config ~/"${!repo_path}/dependencies.yaml"  \
- --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \
+ generate_requirements \
+ "${file}" \
+ --file-key "${repo_keys[$keyi]}" \
+ --output requirements \
+ --config ~/"${!repo_path}/dependencies.yaml" \
+ --matrix "${matrix_selectors}"  \
  ;
  done
 
@@ -103,12 +129,12 @@ make_pip_dependencies() {
  for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do
  local file="/tmp/${!repo_name}.lib${!cpp_name}.${repo_keys[$keyi]}.requirements.txt";
  pip_reqs_txts+=("${file}");
- generate_requirements  \
- "${file}"  \
- --file-key "${repo_keys[$keyi]}"  \
- --output requirements  \
- --config ~/"${!repo_path}/dependencies.yaml"  \
- --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \
+ generate_requirements \
+ "${file}" \
+ --file-key "${repo_keys[$keyi]}" \
+ --output requirements \
+ --config ~/"${!repo_path}/dependencies.yaml" \
+ --matrix "${matrix_selectors}"  \
  ;
  done
  done
@@ -126,12 +152,12 @@ make_pip_dependencies() {
  for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do
  local file="/tmp/${!repo_name}.${!py_name}.${repo_keys[$keyi]}.requirements.txt";
  pip_reqs_txts+=("${file}");
- generate_requirements  \
- "${file}"  \
- --file-key "${repo_keys[$keyi]}"  \
- --output requirements  \
- --config ~/"${!repo_path}/dependencies.yaml"  \
- --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \
+ generate_requirements \
+ "${file}" \
+ --file-key "${repo_keys[$keyi]}" \
+ --output requirements \
+ --config ~/"${!repo_path}/dependencies.yaml" \
+ --matrix "${matrix_selectors}"  \
  ;
  done
  done