use rmm/librmm wheels from rapidsai/rmm#1644, temporarily skip conda …

…jobs
msarahan · Aug 9, 2024 · 0ccfd06 · 0ccfd06
1 parent 4052116
commit 0ccfd06
Show file tree

Hide file tree

Showing 4 changed files with 158 additions and 90 deletions.
diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -13,24 +13,24 @@ jobs:
   pr-builder:
     needs:
       - checks
-      - conda-cpp-build
-      - conda-cpp-checks
-      - conda-cpp-tests
-      - conda-python-build
-      - conda-python-cudf-tests
-      - conda-python-other-tests
-      - conda-java-tests
+      # - conda-cpp-build
+      # - conda-cpp-checks
+      # - conda-cpp-tests
+      # - conda-python-build
+      # - conda-python-cudf-tests
+      # - conda-python-other-tests
+      # - conda-java-tests
       - static-configure
-      - conda-notebook-tests
-      - docs-build
+      # - conda-notebook-tests
+      # - docs-build
       - wheel-build-cpp
       - wheel-build-python
       - wheel-tests-python
       - wheel-build-cudf-polars
       - wheel-tests-cudf-polars
       - wheel-build-dask-cudf
       - wheel-tests-dask-cudf
-      - devcontainer
+      # - devcontainer
       - unit-tests-cudf-pandas
       - pandas-tests
       - pandas-tests-diff
@@ -41,56 +41,56 @@ jobs:
     uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10
     with:
       enable_check_generated_files: false
-  conda-cpp-build:
-    needs: checks
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10
-    with:
-      build_type: pull-request
-  conda-cpp-checks:
-    needs: conda-cpp-build
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10
-    with:
-      build_type: pull-request
-      enable_check_symbols: true
-  conda-cpp-tests:
-    needs: conda-cpp-build
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10
-    with:
-      build_type: pull-request
-  conda-python-build:
-    needs: conda-cpp-build
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10
-    with:
-      build_type: pull-request
-  conda-python-cudf-tests:
-    needs: conda-python-build
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10
-    with:
-      build_type: pull-request
-      script: "ci/test_python_cudf.sh"
-  conda-python-other-tests:
-    # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism
-    needs: conda-python-build
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10
-    with:
-      build_type: pull-request
-      script: "ci/test_python_other.sh"
-  conda-java-tests:
-    needs: conda-cpp-build
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10
-    with:
-      build_type: pull-request
-      node_type: "gpu-v100-latest-1"
-      arch: "amd64"
-      container_image: "rapidsai/ci-conda:latest"
-      run_script: "ci/test_java.sh"
+  # conda-cpp-build:
+  #   needs: checks
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10
+  #   with:
+  #     build_type: pull-request
+  # conda-cpp-checks:
+  #   needs: conda-cpp-build
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10
+  #   with:
+  #     build_type: pull-request
+  #     enable_check_symbols: true
+  # conda-cpp-tests:
+  #   needs: conda-cpp-build
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10
+  #   with:
+  #     build_type: pull-request
+  # conda-python-build:
+  #   needs: conda-cpp-build
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10
+  #   with:
+  #     build_type: pull-request
+  # conda-python-cudf-tests:
+  #   needs: conda-python-build
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10
+  #   with:
+  #     build_type: pull-request
+  #     script: "ci/test_python_cudf.sh"
+  # conda-python-other-tests:
+  #   # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism
+  #   needs: conda-python-build
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10
+  #   with:
+  #     build_type: pull-request
+  #     script: "ci/test_python_other.sh"
+  # conda-java-tests:
+  #   needs: conda-cpp-build
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10
+  #   with:
+  #     build_type: pull-request
+  #     node_type: "gpu-v100-latest-1"
+  #     arch: "amd64"
+  #     container_image: "rapidsai/ci-conda:latest"
+  #     run_script: "ci/test_java.sh"
   static-configure:
     needs: checks
     secrets: inherit
@@ -101,26 +101,26 @@ jobs:
       # primary static consumers (Spark) are not in conda anyway.
       container_image: "rapidsai/ci-wheel:latest"
       run_script: "ci/configure_cpp_static.sh"
-  conda-notebook-tests:
-    needs: conda-python-build
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10
-    with:
-      build_type: pull-request
-      node_type: "gpu-v100-latest-1"
-      arch: "amd64"
-      container_image: "rapidsai/ci-conda:latest"
-      run_script: "ci/test_notebooks.sh"
-  docs-build:
-    needs: conda-python-build
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10
-    with:
-      build_type: pull-request
-      node_type: "gpu-v100-latest-1"
-      arch: "amd64"
-      container_image: "rapidsai/ci-conda:latest"
-      run_script: "ci/build_docs.sh"
+  # conda-notebook-tests:
+  #   needs: conda-python-build
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10
+  #   with:
+  #     build_type: pull-request
+  #     node_type: "gpu-v100-latest-1"
+  #     arch: "amd64"
+  #     container_image: "rapidsai/ci-conda:latest"
+  #     run_script: "ci/test_notebooks.sh"
+  # docs-build:
+  #   needs: conda-python-build
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10
+  #   with:
+  #     build_type: pull-request
+  #     node_type: "gpu-v100-latest-1"
+  #     arch: "amd64"
+  #     container_image: "rapidsai/ci-conda:latest"
+  #     run_script: "ci/build_docs.sh"
   wheel-build-cpp:
     needs: checks
     secrets: inherit
@@ -181,16 +181,16 @@ jobs:
       matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]))
       build_type: pull-request
       script: ci/test_wheel_dask_cudf.sh
-  devcontainer:
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.10
-    with:
-      arch: '["amd64"]'
-      cuda: '["12.5"]'
-      build_command: |
-        sccache -z;
-        build-all -DBUILD_BENCHMARKS=ON --verbose;
-        sccache -s;
+  # devcontainer:
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.10
+  #   with:
+  #     arch: '["amd64"]'
+  #     cuda: '["12.5"]'
+  #     build_command: |
+  #       sccache -z;
+  #       build-all -DBUILD_BENCHMARKS=ON --verbose;
+  #       sccache -s;
   unit-tests-cudf-pandas:
     needs: wheel-build-python
     secrets: inherit

diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh
@@ -5,6 +5,28 @@ set -euo pipefail
 
 package_dir="python/libcudf"
 
+# --- start of section to remove ---#
+# TODO: remove this before merging
+# use librmm and rmm from
+RAPIDS_REPOSITORY=rmm \
+RAPIDS_BUILD_TYPE=pull-requst \
+RAPIDS_REF_NAME=1644 \
+RAPIDS_SHA=0701559 \
+RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" \
+    rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist
+
+RAPIDS_REPOSITORY=rmm \
+RAPIDS_BUILD_TYPE=pull-requst \
+RAPIDS_REF_NAME=1644 \
+RAPIDS_SHA=0701559 \
+RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \
+    rapids-download-wheels-from-s3 python /tmp/libcudf_dist
+
+echo "librmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/librmm_*.whl)" >> /tmp/constraints.txt
+echo "rmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/librmm_*.whl)" >> /tmp/constraints.txt
+export PIP_CONSTRAINT=/tmp/constraints.txt
+# --- end of section to remove ---#
+
 ./ci/build_wheel.sh ${package_dir}
 
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"

diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh
@@ -18,6 +18,28 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist
 
 echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/libcudf_*.whl)" > /tmp/constraints.txt
+
+# --- start of section to remove ---#
+# TODO: remove this before merging
+# use librmm and rmm from
+RAPIDS_REPOSITORY=rmm \
+RAPIDS_BUILD_TYPE=pull-requst \
+RAPIDS_REF_NAME=1644 \
+RAPIDS_SHA=0701559 \
+RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" \
+    rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist
+
+RAPIDS_REPOSITORY=rmm \
+RAPIDS_BUILD_TYPE=pull-requst \
+RAPIDS_REF_NAME=1644 \
+RAPIDS_SHA=0701559 \
+RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \
+    rapids-download-wheels-from-s3 python /tmp/libcudf_dist
+
+echo "librmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/librmm_*.whl)" >> /tmp/constraints.txt
+echo "rmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/librmm_*.whl)" >> /tmp/constraints.txt
+# --- end of section to remove ---#
+
 export PIP_CONSTRAINT="/tmp/constraints.txt"
 ./ci/build_wheel.sh ${package_dir}
 

diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh
@@ -38,6 +38,30 @@ else
     RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
     RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-cudf-dep
     RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./local-cudf-dep
+
+    # --- start of section to remove ---#
+    # TODO: remove this before merging
+    # use librmm and rmm from
+    RAPIDS_REPOSITORY=rmm \
+    RAPIDS_BUILD_TYPE=pull-requst \
+    RAPIDS_REF_NAME=1644 \
+    RAPIDS_SHA=0701559 \
+    RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" \
+        rapids-download-wheels-from-s3 cpp /tmp/local-rmm-dep
+
+    RAPIDS_REPOSITORY=rmm \
+    RAPIDS_BUILD_TYPE=pull-requst \
+    RAPIDS_REF_NAME=1644 \
+    RAPIDS_SHA=0701559 \
+    RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \
+        rapids-download-wheels-from-s3 python /tmp/local-rmm-dep
+
+    echo "librmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/local-rmm-dep/librmm_*.whl)" >> /tmp/constraints.txt
+    echo "rmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/local-rmm-dep/librmm_*.whl)" >> /tmp/constraints.txt
+
+    export PIP_CONSTRAINT=/tmp/constraints.txt
+    # --- end of section to remove ---#
+
     python -m pip install "$(echo ./local-cudf-dep/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)"
     python -m pip install --find-links $(pwd)/local-cudf-dep "$(echo ./local-cudf-dep/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test,cudf-pandas-tests]"
 fi