diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 1ac61006d9f..82d80a7aa5a 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -13,16 +13,16 @@ jobs: pr-builder: needs: - checks - - conda-cpp-build - - conda-cpp-checks - - conda-cpp-tests - - conda-python-build - - conda-python-cudf-tests - - conda-python-other-tests - - conda-java-tests + # - conda-cpp-build + # - conda-cpp-checks + # - conda-cpp-tests + # - conda-python-build + # - conda-python-cudf-tests + # - conda-python-other-tests + # - conda-java-tests - static-configure - - conda-notebook-tests - - docs-build + # - conda-notebook-tests + # - docs-build - wheel-build-cpp - wheel-build-python - wheel-tests-python @@ -30,7 +30,7 @@ jobs: - wheel-tests-cudf-polars - wheel-build-dask-cudf - wheel-tests-dask-cudf - - devcontainer + # - devcontainer - unit-tests-cudf-pandas - pandas-tests - pandas-tests-diff @@ -41,56 +41,56 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 with: enable_check_generated_files: false - conda-cpp-build: - needs: checks - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 - with: - build_type: pull-request - conda-cpp-checks: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 - with: - build_type: pull-request - enable_check_symbols: true - conda-cpp-tests: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 - with: - build_type: pull-request - conda-python-build: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 - with: - build_type: pull-request - conda-python-cudf-tests: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 - with: - build_type: pull-request - script: "ci/test_python_cudf.sh" - conda-python-other-tests: - # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 - with: - build_type: pull-request - script: "ci/test_python_other.sh" - conda-java-tests: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/test_java.sh" + # conda-cpp-build: + # needs: checks + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 + # with: + # build_type: pull-request + # conda-cpp-checks: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 + # with: + # build_type: pull-request + # enable_check_symbols: true + # conda-cpp-tests: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 + # with: + # build_type: pull-request + # conda-python-build: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 + # with: + # build_type: pull-request + # conda-python-cudf-tests: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + # with: + # build_type: pull-request + # script: "ci/test_python_cudf.sh" + # conda-python-other-tests: + # # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + # with: + # build_type: pull-request + # script: "ci/test_python_other.sh" + # conda-java-tests: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + # with: + # build_type: pull-request + # node_type: "gpu-v100-latest-1" + # arch: "amd64" + # container_image: "rapidsai/ci-conda:latest" + # run_script: "ci/test_java.sh" static-configure: needs: checks secrets: inherit @@ -101,26 +101,26 @@ jobs: # primary static consumers (Spark) are not in conda anyway. container_image: "rapidsai/ci-wheel:latest" run_script: "ci/configure_cpp_static.sh" - conda-notebook-tests: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/test_notebooks.sh" - docs-build: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/build_docs.sh" + # conda-notebook-tests: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + # with: + # build_type: pull-request + # node_type: "gpu-v100-latest-1" + # arch: "amd64" + # container_image: "rapidsai/ci-conda:latest" + # run_script: "ci/test_notebooks.sh" + # docs-build: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + # with: + # build_type: pull-request + # node_type: "gpu-v100-latest-1" + # arch: "amd64" + # container_image: "rapidsai/ci-conda:latest" + # run_script: "ci/build_docs.sh" wheel-build-cpp: needs: checks secrets: inherit @@ -181,16 +181,16 @@ jobs: matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: pull-request script: ci/test_wheel_dask_cudf.sh - devcontainer: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.10 - with: - arch: '["amd64"]' - cuda: '["12.5"]' - build_command: | - sccache -z; - build-all -DBUILD_BENCHMARKS=ON --verbose; - sccache -s; + # devcontainer: + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.10 + # with: + # arch: '["amd64"]' + # cuda: '["12.5"]' + # build_command: | + # sccache -z; + # build-all -DBUILD_BENCHMARKS=ON --verbose; + # sccache -s; unit-tests-cudf-pandas: needs: wheel-build-python secrets: inherit diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 9694c3f6144..3b2269dd0cc 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -5,6 +5,28 @@ set -euo pipefail package_dir="python/libcudf" +# --- start of section to remove ---# +# TODO: remove this before merging +# use librmm and rmm from +RAPIDS_REPOSITORY=rmm \ +RAPIDS_BUILD_TYPE=pull-requst \ +RAPIDS_REF_NAME=1644 \ +RAPIDS_SHA=0701559 \ +RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" \ + rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist + +RAPIDS_REPOSITORY=rmm \ +RAPIDS_BUILD_TYPE=pull-requst \ +RAPIDS_REF_NAME=1644 \ +RAPIDS_SHA=0701559 \ +RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ + rapids-download-wheels-from-s3 python /tmp/libcudf_dist + +echo "librmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/librmm_*.whl)" >> /tmp/constraints.txt +echo "rmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/librmm_*.whl)" >> /tmp/constraints.txt +export PIP_CONSTRAINT=/tmp/constraints.txt +# --- end of section to remove ---# + ./ci/build_wheel.sh ${package_dir} RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 2c1bb198c06..222a56fc375 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -18,6 +18,28 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/libcudf_*.whl)" > /tmp/constraints.txt + +# --- start of section to remove ---# +# TODO: remove this before merging +# use librmm and rmm from +RAPIDS_REPOSITORY=rmm \ +RAPIDS_BUILD_TYPE=pull-requst \ +RAPIDS_REF_NAME=1644 \ +RAPIDS_SHA=0701559 \ +RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" \ + rapids-download-wheels-from-s3 cpp /tmp/libcudf_dist + +RAPIDS_REPOSITORY=rmm \ +RAPIDS_BUILD_TYPE=pull-requst \ +RAPIDS_REF_NAME=1644 \ +RAPIDS_SHA=0701559 \ +RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ + rapids-download-wheels-from-s3 python /tmp/libcudf_dist + +echo "librmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/librmm_*.whl)" >> /tmp/constraints.txt +echo "rmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcudf_dist/librmm_*.whl)" >> /tmp/constraints.txt +# --- end of section to remove ---# + export PIP_CONSTRAINT="/tmp/constraints.txt" ./ci/build_wheel.sh ${package_dir} diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index b452eabc6ec..bddd30316d2 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -38,6 +38,30 @@ else RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-cudf-dep RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./local-cudf-dep + + # --- start of section to remove ---# + # TODO: remove this before merging + # use librmm and rmm from + RAPIDS_REPOSITORY=rmm \ + RAPIDS_BUILD_TYPE=pull-requst \ + RAPIDS_REF_NAME=1644 \ + RAPIDS_SHA=0701559 \ + RAPIDS_PY_WHEEL_NAME="librmm_${RAPIDS_PY_CUDA_SUFFIX}" \ + rapids-download-wheels-from-s3 cpp /tmp/local-rmm-dep + + RAPIDS_REPOSITORY=rmm \ + RAPIDS_BUILD_TYPE=pull-requst \ + RAPIDS_REF_NAME=1644 \ + RAPIDS_SHA=0701559 \ + RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" \ + rapids-download-wheels-from-s3 python /tmp/local-rmm-dep + + echo "librmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/local-rmm-dep/librmm_*.whl)" >> /tmp/constraints.txt + echo "rmm-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/local-rmm-dep/librmm_*.whl)" >> /tmp/constraints.txt + + export PIP_CONSTRAINT=/tmp/constraints.txt + # --- end of section to remove ---# + python -m pip install "$(echo ./local-cudf-dep/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" python -m pip install --find-links $(pwd)/local-cudf-dep "$(echo ./local-cudf-dep/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test,cudf-pandas-tests]" fi