Bump up CUDA version used for tests

- disable MXNet tests are MXNet is no longer actively developed - extend support for DLPack capsules that provide strides collapsing dimensions Signed-off-by: Janusz Lisiecki <jlisiecki@nvidia.com>
NVIDIA · Oct 25, 2023 · 1da40c8 · 1da40c8
1 parent 15a4956
commit 1da40c8
Show file tree

Hide file tree

Showing 16 changed files with 30 additions and 27 deletions.
diff --git a/dali/python/backend_impl.cc b/dali/python/backend_impl.cc
@@ -155,11 +155,18 @@ void CheckContiguousTensor(const TStrides &strides, int num_strides,
   DALI_ENFORCE(num_strides == num_extents,
     "There should be exactly as many strides as there are extents in array shape.");
   int64_t stride_from_shape = element_size;
+  int64_t stride_from_shape_collapsed = 1;
+  int64_t last_non_one_dim = 1;
   for (int i = num_strides - 1; i >= 0; i--) {
-    DALI_ENFORCE(strides[i] == stride_from_shape,
+    DALI_ENFORCE(strides[i] == stride_from_shape || strides[i] == stride_from_shape_collapsed,
         make_string("Strided data not supported. Dimension ", i, " has stride ", strides[i],
         " whereas densely packed data of this shape would have a stride ", stride_from_shape));
     stride_from_shape *= shape[i];
+    // for shapes [1, 1, 5] leading dimensions may not contribute to
+    if (shape[i] != 1) {
+      stride_from_shape_collapsed *= last_non_one_dim;
+      last_non_one_dim = shape[i];
+    }
   }
 }
 

diff --git a/dali/test/python/test_torch_pipeline_rnnt.py b/dali/test/python/test_torch_pipeline_rnnt.py
@@ -159,7 +159,9 @@ def forward(self, inp, seq_len):
         # do stft
         x = torch.stft(x, n_fft=self.n_fft, hop_length=self.hop_length,
                        win_length=self.win_length, pad_mode='reflect',
-                       center=True, window=self.window.to(dtype=torch.float).to(x.device))
+                       center=True, window=self.window.to(dtype=torch.float).to(x.device),
+                       return_complex=True)
+        x = torch.view_as_real(x)
 
         # get power spectrum
         x = x.pow(2).sum(-1)
@@ -214,7 +216,9 @@ def torch_spectrogram(audio, sample_rate, device='cpu',
     window_tensor = window_fn(win_length, periodic=False) if window_fn else None
     stft_out = torch.stft(audio, n_fft=n_fft, hop_length=hop_length,
                           win_length=win_length, pad_mode=pad_mode,
-                          center=center, window=window_tensor.to(dtype=torch.float))
+                          center=center, window=window_tensor.to(dtype=torch.float),
+                          return_complex=True)
+    stft_out = torch.view_as_real(stft_out)
     # get power spectrum
     spectrogram = stft_out.pow(2).sum(-1)
     spectrogram = spectrogram.cpu().numpy()

diff --git a/qa/TL0_FW_iterators/test.sh b/qa/TL0_FW_iterators/test.sh
@@ -1,6 +1,5 @@
 #!/bin/bash -e
 bash -e ./test_tf.sh
 bash -e ./test_paddle.sh
-bash -e ./test_mxnet.sh
 bash -e ./test_pytorch.sh
 bash -e ./test_jax.sh
diff --git a/qa/TL0_FW_iterators/test_mxnet.sh b/qa/TL0_FW_iterators/test_mxnet.sh
@@ -1,6 +1,6 @@
 #!/bin/bash -e
 # used pip packages
-pip_packages='${python_test_runner_package} numpy mxnet'
+pip_packages='${python_test_runner_package} numpy'
 target_dir=./dali/test/python
 
 one_config_only=true

diff --git a/qa/TL0_framework_imports/test.sh b/qa/TL0_framework_imports/test.sh
@@ -1,6 +1,5 @@
 #!/bin/bash -e
 bash -e ./test_tf.sh
-bash -e ./test_mxnet.sh
 bash -e ./test_pytorch.sh
 bash -e ./test_jax.sh
 bash -e ./test_no_fw.sh
diff --git a/qa/TL0_framework_imports/test_mxnet.sh b/qa/TL0_framework_imports/test_mxnet.sh
@@ -1,7 +1,5 @@
 #!/bin/bash -e
-# used pip packages
-
-pip_packages='mxnet'
+# to be run inside a MXNet container - so don't need to list it here as a pip package dependency
 
 test_body() {
     # test code

diff --git a/qa/TL0_python_self_test_frameworks/test.sh b/qa/TL0_python_self_test_frameworks/test.sh
@@ -1,5 +1,4 @@
 #!/bin/bash -e
-./test_mxnet.sh
 ./test_cupy.sh
 ./test_pytorch.sh
 ./test_pytorch_cupy.sh
diff --git a/qa/TL0_python_self_test_frameworks/test_mxnet.sh b/qa/TL0_python_self_test_frameworks/test_mxnet.sh
@@ -1,6 +1,7 @@
 #!/bin/bash -e
 # used pip packages
-pip_packages='${python_test_runner_package} numpy mxnet psutil'
+# to be run inside a MXNet container - so don't need to list it here as a pip package dependency
+pip_packages='${python_test_runner_package} numpy psutil'
 target_dir=./dali/test/python
 
 test_body() {

diff --git a/qa/TL1_custom_src_pattern_build/test.sh b/qa/TL1_custom_src_pattern_build/test.sh
@@ -1,6 +1,7 @@
 #!/bin/bash -e
 
 build_and_check() {
+  pip install black dm-tree
   make -j
   pip install ./dali/python
 

diff --git a/qa/TL1_jupyter_plugins/test.sh b/qa/TL1_jupyter_plugins/test.sh
@@ -1,5 +1,4 @@
 #!/bin/bash -e
-bash -e ./test_mxnet.sh
 bash -e ./test_tf.sh
 bash -e ./test_paddle.sh
 bash -e ./test_pytorch.sh
diff --git a/qa/TL1_jupyter_plugins/test_mxnet.sh b/qa/TL1_jupyter_plugins/test_mxnet.sh
@@ -1,7 +1,8 @@
 #!/bin/bash -e
 
 # used pip packages
-pip_packages='jupyter matplotlib<3.5.3 mxnet'
+# to be run inside a MXNet container - so don't need to list it here as a pip package dependency
+pip_packages='jupyter matplotlib<3.5.3'
 target_dir=./docs/examples/
 
 do_once() {

diff --git a/qa/TL1_separate_executor/test.sh b/qa/TL1_separate_executor/test.sh
@@ -2,5 +2,4 @@
 bash -e ./test_nofw.sh
 bash -e ./test_tf.sh
 bash -e ./test_pytorch.sh
-bash -e ./test_mxnet.sh
 bash -e ./test_paddle.sh
diff --git a/qa/TL1_separate_executor/test_mxnet.sh b/qa/TL1_separate_executor/test_mxnet.sh
@@ -1,6 +1,6 @@
 #!/bin/bash -e
 # used pip packages
-pip_packages='torch mxnet'
+# to be run inside a MXNet container - so don't need to list it here as a pip package dependency
 target_dir=./dali/test/python
 one_config_only=true
 

diff --git a/qa/TL2_FW_iterators_perf/test.sh b/qa/TL2_FW_iterators_perf/test.sh
@@ -1,5 +1,4 @@
 #!/bin/bash -e
 bash -e ./test_tf.sh
 bash -e ./test_paddle.sh
-bash -e ./test_mxnet.sh
 bash -e ./test_pytorch.sh
diff --git a/qa/TL2_FW_iterators_perf/test_mxnet.sh b/qa/TL2_FW_iterators_perf/test_mxnet.sh
@@ -1,6 +1,6 @@
 #!/bin/bash -e
 # used pip packages
-pip_packages='mxnet'
+# to be run inside a MXNet container - so don't need to list it here as a pip package dependency
 target_dir=./dali/test/python
 one_config_only=true
 

diff --git a/qa/setup_packages.py b/qa/setup_packages.py
@@ -465,12 +465,9 @@ def get_pyvers_name(self, url, cuda_version):
 all_packages = [PlainPackage("numpy", [">=1.17,<1.24"]),
                 PlainPackage("opencv-python", [PckgVer("4.5.4.60", dependencies=["numpy<1.24"])]),
                 CudaPackage("cupy",
-                            {"113": [PckgVer("10.0.0", python_min_ver="3.8",
+                            {"118": [PckgVer("12.2.0", python_min_ver="3.8",
                                              dependencies=["numpy<1.24"])]},
-                            "cupy-cuda{cuda_v}"),
-                CudaPackage("mxnet",
-                            {"113": [PckgVer("1.9.1", dependencies=["numpy<1.24"])]},
-                            "mxnet-cu{cuda_v}"),
+                            "cupy-cuda11x"),
                 CudaPackage("tensorflow-gpu",
                             {"110": [
                                 PckgVer("2.12.1", python_min_ver="3.8", alias="tensorflow",
@@ -480,20 +477,20 @@ def get_pyvers_name(self, url, cuda_version):
                                         dependencies=["protobuf<4", "numpy<1.24",
                                                       "urllib3<2.0", "typing_extensions<4.6"])]}),
                 CudaPackageExtraIndex("torch",
-                                      {"113": [PckgVer("1.11.0", python_min_ver="3.8",
+                                      {"118": [PckgVer("2.1.0", python_min_ver="3.8",
                                                        dependencies=["numpy<1.24"])]},
                                       extra_index="https://download.pytorch.org/whl/cu{cuda_v}/"),
                 CudaPackageExtraIndex("torchvision",
-                                      {"113": [PckgVer("0.12.0", python_min_ver="3.8",
+                                      {"118": [PckgVer("0.16.0", python_min_ver="3.8",
                                                        dependencies=["numpy<1.24"])]},
                                       extra_index="https://download.pytorch.org/whl/cu{cuda_v}/"),
                 CudaPackageExtraIndex("paddlepaddle-gpu",
-                                      {"110": [PckgVer("2.4.1.post117",
+                                      {"110": [PckgVer("2.5.0.post118",
                                                        dependencies=["protobuf<4", "numpy<1.24"])]},
                                       links_index="https://www.paddlepaddle.org.cn/"
                                                   "whl/linux/mkl/avx/stable.html"),
-                CudaPackageExtraIndex("jax",  # name used in our test script, see the mxnet case
-                                      {"113": [PckgVer("0.4.13", python_min_ver="3.8",
+                CudaPackageExtraIndex("jax",  # name used in our test script
+                                      {"118": [PckgVer("0.4.13", python_min_ver="3.8",
                                                        dependencies=["jaxlib"])]},
                                       # name used during installation
                                       name="jax[cuda{cuda_v[0]}{cuda_v[1]}_local]",