Bump up CUDA version used for tests

- disable MXNet tests are MXNet is no longer actively developed Signed-off-by: Janusz Lisiecki <jlisiecki@nvidia.com>
NVIDIA · Oct 23, 2023 · b37b2c8 · b37b2c8
1 parent 15a4956
commit b37b2c8
Show file tree

Hide file tree

Showing 13 changed files with 114 additions and 123 deletions.
diff --git a/qa/TL0_FW_iterators/test.sh b/qa/TL0_FW_iterators/test.sh
@@ -1,6 +1,5 @@
 #!/bin/bash -e
 bash -e ./test_tf.sh
 bash -e ./test_paddle.sh
-bash -e ./test_mxnet.sh
 bash -e ./test_pytorch.sh
 bash -e ./test_jax.sh
diff --git a/qa/TL0_FW_iterators/test_mxnet.sh b/qa/TL0_FW_iterators/test_mxnet.sh
@@ -1,29 +1,29 @@
-#!/bin/bash -e
-# used pip packages
-pip_packages='${python_test_runner_package} numpy mxnet'
-target_dir=./dali/test/python
+# #!/bin/bash -e
+# # used pip packages
+# pip_packages='${python_test_runner_package} numpy mxnet'
+# target_dir=./dali/test/python
 
-one_config_only=true
+# one_config_only=true
 
-do_once() {
-    NUM_GPUS=$(nvidia-smi -L | wc -l)
-}
+# do_once() {
+#     NUM_GPUS=$(nvidia-smi -L | wc -l)
+# }
 
-test_body() {
-    # it takes very long time to run it with sanitizers on and provides little value so turn it off
-    if [ -z "$DALI_ENABLE_SANITIZERS" ]; then
-        for fw in "mxnet"; do
-            python test_RN50_data_fw_iterators.py --framework ${fw} --gpus ${NUM_GPUS} -b 13 \
-                --workers 3 --prefetch 2 -i 100 --epochs 2
-            python test_RN50_data_fw_iterators.py --framework ${fw} --gpus ${NUM_GPUS} -b 13 \
-                --workers 3 --prefetch 2 -i 2 --epochs 2 --fp16
-        done
-    fi
-    ${python_invoke_test} -m '(?:^|[\b_\./-])[Tt]est.*mxnet*' test_fw_iterators_detection.py
-    ${python_invoke_test} -m '(?:^|[\b_\./-])[Tt]est.*mxnet*' test_fw_iterators.py
-    ${python_invoke_test} -m '(?:^|[\b_\./-])[Tt]est.*gluon*' test_fw_iterators.py
-}
+# test_body() {
+#     # it takes very long time to run it with sanitizers on and provides little value so turn it off
+#     if [ -z "$DALI_ENABLE_SANITIZERS" ]; then
+#         for fw in "mxnet"; do
+#             python test_RN50_data_fw_iterators.py --framework ${fw} --gpus ${NUM_GPUS} -b 13 \
+#                 --workers 3 --prefetch 2 -i 100 --epochs 2
+#             python test_RN50_data_fw_iterators.py --framework ${fw} --gpus ${NUM_GPUS} -b 13 \
+#                 --workers 3 --prefetch 2 -i 2 --epochs 2 --fp16
+#         done
+#     fi
+#     ${python_invoke_test} -m '(?:^|[\b_\./-])[Tt]est.*mxnet*' test_fw_iterators_detection.py
+#     ${python_invoke_test} -m '(?:^|[\b_\./-])[Tt]est.*mxnet*' test_fw_iterators.py
+#     ${python_invoke_test} -m '(?:^|[\b_\./-])[Tt]est.*gluon*' test_fw_iterators.py
+# }
 
-pushd ../..
-source ./qa/test_template.sh
-popd
+# pushd ../..
+# source ./qa/test_template.sh
+# popd
diff --git a/qa/TL0_framework_imports/test.sh b/qa/TL0_framework_imports/test.sh
@@ -1,6 +1,5 @@
 #!/bin/bash -e
 bash -e ./test_tf.sh
-bash -e ./test_mxnet.sh
 bash -e ./test_pytorch.sh
 bash -e ./test_jax.sh
 bash -e ./test_no_fw.sh
diff --git a/qa/TL0_framework_imports/test_mxnet.sh b/qa/TL0_framework_imports/test_mxnet.sh
@@ -1,16 +1,16 @@
-#!/bin/bash -e
-# used pip packages
+# #!/bin/bash -e
+# # used pip packages
 
-pip_packages='mxnet'
+# pip_packages='mxnet'
 
-test_body() {
-    # test code
-    echo "---------Testing MXNET;DALI----------"
-    ( set -x && python -c "import mxnet; import nvidia.dali.plugin.mxnet" )
-    echo "---------Testing DALI;MXNET----------"
-    ( set -x && python -c "import nvidia.dali.plugin.mxnet; import mxnet" )
-}
+# test_body() {
+#     # test code
+#     echo "---------Testing MXNET;DALI----------"
+#     ( set -x && python -c "import mxnet; import nvidia.dali.plugin.mxnet" )
+#     echo "---------Testing DALI;MXNET----------"
+#     ( set -x && python -c "import nvidia.dali.plugin.mxnet; import mxnet" )
+# }
 
-pushd ../../
-source ./qa/test_template.sh
-popd
+# pushd ../../
+# source ./qa/test_template.sh
+# popd
diff --git a/qa/TL0_python_self_test_frameworks/test.sh b/qa/TL0_python_self_test_frameworks/test.sh
@@ -1,5 +1,4 @@
 #!/bin/bash -e
-./test_mxnet.sh
 ./test_cupy.sh
 ./test_pytorch.sh
 ./test_pytorch_cupy.sh
diff --git a/qa/TL0_python_self_test_frameworks/test_mxnet.sh b/qa/TL0_python_self_test_frameworks/test_mxnet.sh
@@ -1,15 +1,15 @@
-#!/bin/bash -e
-# used pip packages
-pip_packages='${python_test_runner_package} numpy mxnet psutil'
-target_dir=./dali/test/python
+# #!/bin/bash -e
+# # used pip packages
+# pip_packages='${python_test_runner_package} numpy mxnet psutil'
+# target_dir=./dali/test/python
 
-test_body() {
-    ${python_invoke_test} -m '(?:^|[\b_\./-])[Tt]est.*mxnet' test_dltensor_operator.py
-    ${python_invoke_test} test_external_source_parallel_mxnet.py
-    ${python_invoke_test} --attr 'mxnet' test_external_source_impl_utils.py
-    ${python_invoke_test} --attr 'mxnet' test_pipeline_debug.py
-}
+# test_body() {
+#     ${python_invoke_test} -m '(?:^|[\b_\./-])[Tt]est.*mxnet' test_dltensor_operator.py
+#     ${python_invoke_test} test_external_source_parallel_mxnet.py
+#     ${python_invoke_test} --attr 'mxnet' test_external_source_impl_utils.py
+#     ${python_invoke_test} --attr 'mxnet' test_pipeline_debug.py
+# }
 
-pushd ../..
-source ./qa/test_template.sh
-popd
+# pushd ../..
+# source ./qa/test_template.sh
+# popd
diff --git a/qa/TL1_jupyter_plugins/test.sh b/qa/TL1_jupyter_plugins/test.sh
@@ -1,5 +1,4 @@
 #!/bin/bash -e
-bash -e ./test_mxnet.sh
 bash -e ./test_tf.sh
 bash -e ./test_paddle.sh
 bash -e ./test_pytorch.sh
diff --git a/qa/TL1_jupyter_plugins/test_mxnet.sh b/qa/TL1_jupyter_plugins/test_mxnet.sh
@@ -1,27 +1,27 @@
-#!/bin/bash -e
+# #!/bin/bash -e
 
-# used pip packages
-pip_packages='jupyter matplotlib<3.5.3 mxnet'
-target_dir=./docs/examples/
+# # used pip packages
+# pip_packages='jupyter matplotlib<3.5.3 mxnet'
+# target_dir=./docs/examples/
 
-do_once() {
-  mkdir -p idx_files
-}
+# do_once() {
+#   mkdir -p idx_files
+# }
 
-test_body() {
-  # dummy
-  exclude_files="#"
+# test_body() {
+#   # dummy
+#   exclude_files="#"
 
-  # test code
-  find frameworks/mxnet -name "*.ipynb" | sed "/${exclude_files}/d" | xargs -i jupyter nbconvert \
-                  --to notebook --inplace --execute \
-                  --ExecutePreprocessor.kernel_name=python${PYVER:0:1} \
-                  --ExecutePreprocessor.timeout=600 {}
-  jupyter nbconvert --to notebook --inplace --execute \
-                    --ExecutePreprocessor.kernel_name=python${PYVER:0:1} \
-                    --ExecutePreprocessor.timeout=600 use_cases/mxnet/mxnet-resnet50.ipynb
-}
+#   # test code
+#   find frameworks/mxnet -name "*.ipynb" | sed "/${exclude_files}/d" | xargs -i jupyter nbconvert \
+#                   --to notebook --inplace --execute \
+#                   --ExecutePreprocessor.kernel_name=python${PYVER:0:1} \
+#                   --ExecutePreprocessor.timeout=600 {}
+#   jupyter nbconvert --to notebook --inplace --execute \
+#                     --ExecutePreprocessor.kernel_name=python${PYVER:0:1} \
+#                     --ExecutePreprocessor.timeout=600 use_cases/mxnet/mxnet-resnet50.ipynb
+# }
 
-pushd ../..
-source ./qa/test_template.sh
-popd
+# pushd ../..
+# source ./qa/test_template.sh
+# popd
diff --git a/qa/TL1_separate_executor/test.sh b/qa/TL1_separate_executor/test.sh
@@ -2,5 +2,4 @@
 bash -e ./test_nofw.sh
 bash -e ./test_tf.sh
 bash -e ./test_pytorch.sh
-bash -e ./test_mxnet.sh
 bash -e ./test_paddle.sh
diff --git a/qa/TL1_separate_executor/test_mxnet.sh b/qa/TL1_separate_executor/test_mxnet.sh
@@ -1,20 +1,20 @@
-#!/bin/bash -e
-# used pip packages
-pip_packages='torch mxnet'
-target_dir=./dali/test/python
-one_config_only=true
+# #!/bin/bash -e
+# # used pip packages
+# pip_packages='torch mxnet'
+# target_dir=./dali/test/python
+# one_config_only=true
 
-do_once() {
-    NUM_GPUS=$(nvidia-smi -L | wc -l)
-}
+# do_once() {
+#     NUM_GPUS=$(nvidia-smi -L | wc -l)
+# }
 
-test_body() {
-    for fw in "mxnet"; do
-        python test_RN50_data_fw_iterators.py --framework ${fw} --gpus ${NUM_GPUS} -b 13 \
-            --workers 3 --separate_queue --cpu_size 3 --gpu_size 2 --iters 32 --epochs 2
-    done
-}
+# test_body() {
+#     for fw in "mxnet"; do
+#         python test_RN50_data_fw_iterators.py --framework ${fw} --gpus ${NUM_GPUS} -b 13 \
+#             --workers 3 --separate_queue --cpu_size 3 --gpu_size 2 --iters 32 --epochs 2
+#     done
+# }
 
-pushd ../..
-source ./qa/test_template.sh
-popd
+# pushd ../..
+# source ./qa/test_template.sh
+# popd
diff --git a/qa/TL2_FW_iterators_perf/test.sh b/qa/TL2_FW_iterators_perf/test.sh
@@ -1,5 +1,4 @@
 #!/bin/bash -e
 bash -e ./test_tf.sh
 bash -e ./test_paddle.sh
-bash -e ./test_mxnet.sh
 bash -e ./test_pytorch.sh
diff --git a/qa/TL2_FW_iterators_perf/test_mxnet.sh b/qa/TL2_FW_iterators_perf/test_mxnet.sh
@@ -1,20 +1,20 @@
-#!/bin/bash -e
-# used pip packages
-pip_packages='mxnet'
-target_dir=./dali/test/python
-one_config_only=true
+# #!/bin/bash -e
+# # used pip packages
+# pip_packages='mxnet'
+# target_dir=./dali/test/python
+# one_config_only=true
 
-do_once() {
-    NUM_GPUS=$(nvidia-smi -L | wc -l)
-}
+# do_once() {
+#     NUM_GPUS=$(nvidia-smi -L | wc -l)
+# }
 
-test_body() {
-    for fw in "mxnet"; do
-        python test_RN50_data_fw_iterators.py --framework ${fw} --gpus ${NUM_GPUS} -b 13 \
-            --workers 3 --prefetch 2 --epochs 3
-    done
-}
+# test_body() {
+#     for fw in "mxnet"; do
+#         python test_RN50_data_fw_iterators.py --framework ${fw} --gpus ${NUM_GPUS} -b 13 \
+#             --workers 3 --prefetch 2 --epochs 3
+#     done
+# }
 
-pushd ../..
-source ./qa/test_template.sh
-popd
+# pushd ../..
+# source ./qa/test_template.sh
+# popd
diff --git a/qa/setup_packages.py b/qa/setup_packages.py
@@ -465,12 +465,9 @@ def get_pyvers_name(self, url, cuda_version):
 all_packages = [PlainPackage("numpy", [">=1.17,<1.24"]),
                 PlainPackage("opencv-python", [PckgVer("4.5.4.60", dependencies=["numpy<1.24"])]),
                 CudaPackage("cupy",
-                            {"113": [PckgVer("10.0.0", python_min_ver="3.8",
+                            {"118": [PckgVer("12.2.0", python_min_ver="3.8",
                                              dependencies=["numpy<1.24"])]},
-                            "cupy-cuda{cuda_v}"),
-                CudaPackage("mxnet",
-                            {"113": [PckgVer("1.9.1", dependencies=["numpy<1.24"])]},
-                            "mxnet-cu{cuda_v}"),
+                            "cupy-cuda11x"),
                 CudaPackage("tensorflow-gpu",
                             {"110": [
                                 PckgVer("2.12.1", python_min_ver="3.8", alias="tensorflow",
@@ -480,20 +477,20 @@ def get_pyvers_name(self, url, cuda_version):
                                         dependencies=["protobuf<4", "numpy<1.24",
                                                       "urllib3<2.0", "typing_extensions<4.6"])]}),
                 CudaPackageExtraIndex("torch",
-                                      {"113": [PckgVer("1.11.0", python_min_ver="3.8",
+                                      {"118": [PckgVer("2.0.1", python_min_ver="3.8",
                                                        dependencies=["numpy<1.24"])]},
                                       extra_index="https://download.pytorch.org/whl/cu{cuda_v}/"),
                 CudaPackageExtraIndex("torchvision",
-                                      {"113": [PckgVer("0.12.0", python_min_ver="3.8",
+                                      {"118": [PckgVer("0.15.2", python_min_ver="3.8",
                                                        dependencies=["numpy<1.24"])]},
                                       extra_index="https://download.pytorch.org/whl/cu{cuda_v}/"),
                 CudaPackageExtraIndex("paddlepaddle-gpu",
-                                      {"110": [PckgVer("2.4.1.post117",
+                                      {"110": [PckgVer("2.4.1.post118",
                                                        dependencies=["protobuf<4", "numpy<1.24"])]},
                                       links_index="https://www.paddlepaddle.org.cn/"
                                                   "whl/linux/mkl/avx/stable.html"),
-                CudaPackageExtraIndex("jax",  # name used in our test script, see the mxnet case
-                                      {"113": [PckgVer("0.4.13", python_min_ver="3.8",
+                CudaPackageExtraIndex("jax",  # name used in our test script
+                                      {"118": [PckgVer("0.4.13", python_min_ver="3.8",
                                                        dependencies=["jaxlib"])]},
                                       # name used during installation
                                       name="jax[cuda{cuda_v[0]}{cuda_v[1]}_local]",