Merge branch 'branch-24.04' into branch-24.06

features/src/rapids-build-utils/devcontainer-feature.json

@@ -1,7 +1,7 @@
 {
   "name": "NVIDIA RAPIDS devcontainer build utilities",
   "id": "rapids-build-utils",
-  "version": "24.6.1",
+  "version": "24.6.2",
   "description": "A feature to install the RAPIDS devcontainer build utilities",
   "containerEnv": {
     "BASH_ENV": "/etc/bash.bash_env"

features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh

@@ -56,6 +56,29 @@ make_conda_dependencies() {
 
         if [ -f ~/"${!repo_path}/dependencies.yaml" ]; then
 
+            local cpp_length="${repo}_cpp_length";
+
+            for ((j=0; j < ${!cpp_length:-0}; j+=1)); do
+                local cpp_name="${repo}_cpp_${j}_name";
+
+                echo "Generating lib${!cpp_name}'s conda env yml" 1>&2;
+
+                local repo_keys=("${key[@]}" "${key[@]/%/_lib${!cpp_name//"-"/"_"}}");
+                local keyi;
+
+                for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do
+                    local file="/tmp/${!repo_name}.lib${!cpp_name}.${repo_keys[$keyi]}.env.yaml";
+                    conda_env_yamls+=("${file}");
+                    generate_env_yaml                                                         \
+                        "${file}"                                                             \
+                        --file_key "${repo_keys[$keyi]}"                                      \
+                        --output conda                                                        \
+                        --config ~/"${!repo_path}/dependencies.yaml"                          \
+                        --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \
+                        ;
+                done
+            done
+
             local py_length="${repo}_python_length";
 
             for ((j=0; j < ${!py_length:-0}; j+=1)); do

features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh

@@ -50,7 +50,7 @@ make_conda_env() {
         # If the conda env does exist but it's different from the generated one,
         # print the diff between the envs and update it
         elif ! diff -BNqw "${old_env_path}" "${new_env_path}" >/dev/null 2>&1; then
-            echo -e "Updating '${env_name}' conda environment\n" 1>&2;
+            echo -e "Creating '${env_name}' conda environment\n" 1>&2;
             echo -e "Environment (${env_file_name}):\n" 1>&2;
 
             # Print the diff to the console for debugging
@@ -60,12 +60,18 @@ make_conda_env() {
              || true                                           \
              && echo "";
 
-            # Update the current conda env + prune libs that were removed
-            # Use conda instead of mamba due to https://github.com/mamba-org/mamba/issues/3059
-            conda env update -n "${env_name}" -f "${new_env_path}" --prune --solver=libmamba;
+            # If the conda env exists, recreate it from scratch.
+            # Most conda issues are due to updating existing envs with new packages.
+            # We mount in the package cache, so this should still be fast in most cases.
+            rm -rf "${HOME}/.conda/envs/${env_name}";
+
+            conda env create -n "${env_name}" -f "${new_env_path}" --solver=libmamba;
         fi
 
         cp -a "${new_env_path}" "${old_env_path}";
+    else
+        rm -f "${new_env_path}" "${old_env_path}";
+        echo -e "Not creating '${env_name}' conda environment because '${env_file_name}' is empty." 1>&2;
     fi
 }
 

features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh

@@ -59,6 +59,29 @@ make_pip_dependencies() {
 
         if [ -f ~/"${!repo_path}/dependencies.yaml" ]; then
 
+            local cpp_length="${repo}_cpp_length";
+
+            for ((j=0; j < ${!cpp_length:-0}; j+=1)); do
+                local cpp_name="${repo}_cpp_${j}_name";
+
+                echo "Generating lib${!cpp_name}'s requirements.txt" 1>&2;
+
+                local repo_keys=("${key[@]}" "${key[@]/%/_lib${!cpp_name//"-"/"_"}}");
+                local keyi;
+
+                for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do
+                    local file="/tmp/${!repo_name}.lib${!cpp_name}.${repo_keys[$keyi]}.requirements.txt";
+                    pip_reqs_txts+=("${file}");
+                    generate_requirements                                                     \
+                        "${file}"                                                             \
+                        --file_key "${repo_keys[$keyi]}"                                      \
+                        --output requirements                                                 \
+                        --config ~/"${!repo_path}/dependencies.yaml"                          \
+                        --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \
+                        ;
+                done
+            done
+
             local py_length="${repo}_python_length";
 
             for ((j=0; j < ${!py_length:-0}; j+=1)); do

matrix.yml

@@ -29,8 +29,8 @@ x-llvm-prev: &llvm_17 { name: "llvm", version: "17" }
 x-llvm-curr: &llvm_18 { name: "llvm", version: "18" }
 x-llvm-env: &llvm_env {CC: "clang", CXX: "clang++", CUDAHOSTCXX: "clang++"}
 
-x-nvhpc-prev: &nvhpc_prev { name: "nvhpc", version: "23.11" }
-x-nvhpc-curr: &nvhpc_curr { name: "nvhpc", version: "24.1" }
+x-nvhpc-prev: &nvhpc_prev { name: "nvhpc", version: "24.1" }
+x-nvhpc-curr: &nvhpc_curr { name: "nvhpc", version: "24.3" }
 
 x-nvhpc-env: &nvhpc_env {CC: "nvc", CXX: "nvc++"}