From ad178493d11415f04b0b5bead0cad4689c77fd08 Mon Sep 17 00:00:00 2001
From: Kenichi Maehashi <939877+kmaehashi@users.noreply.github.com>
Date: Wed, 18 Jan 2023 01:45:55 +0900
Subject: [PATCH] Merge pull request #322 from kmaehashi/cuda12x
Support CUDA 12.0
---
.pfnci/build_submit.sh | 4 +-
.pfnci/config.pbtxt | 4 +-
.pfnci/wheel-windows/_flexci.ps1 | 49 +++--
.pfnci/wheel-windows/build.ps1 | 15 +-
build.sh | 4 +
builder/base/cuda-runfile/build.sh | 10 ++
check_release_assets.py | 3 +
dist_config.py | 226 +++++-------------------
release-tests/pkg_wheel/test_preload.py | 4 +
9 files changed, 115 insertions(+), 204 deletions(-)
diff --git a/.pfnci/build_submit.sh b/.pfnci/build_submit.sh
index 03ec65f9..680d873f 100755
--- a/.pfnci/build_submit.sh
+++ b/.pfnci/build_submit.sh
@@ -33,12 +33,12 @@ submit_job() {
submit_job cupy-wheel-linux ".pfnci/wheel-linux/main.sh sdist 3.7 ${BRANCH} ${JOB_GROUP}"
# wheels (Linux)
-for CUDA in 10.2 11.0 11.1 11.x; do
+for CUDA in 10.2 11.0 11.1 11.x 12.x; do
submit_job cupy-wheel-linux ".pfnci/wheel-linux/main.sh ${CUDA} 3.7,3.8,3.9,3.10,3.11 ${BRANCH} ${JOB_GROUP}"
done
# wheels (Windows)
-for CUDA in 10.2 11.0 11.1 11.x; do
+for CUDA in 10.2 11.0 11.1 11.x 12.x; do
for PYTHON in 3.7 3.8 3.9 3.10 3.11; do
submit_job cupy-wheel-win ".pfnci\\wheel-windows\\main.bat ${CUDA} ${PYTHON} ${BRANCH} ${JOB_GROUP}"
done
diff --git a/.pfnci/config.pbtxt b/.pfnci/config.pbtxt
index 9edef8a7..a0cf038b 100644
--- a/.pfnci/config.pbtxt
+++ b/.pfnci/config.pbtxt
@@ -10,7 +10,7 @@ configs {
time_limit {
seconds: 10800
}
- command: ".pfnci/wheel-linux/main.sh 11.x 3.7"
+ command: ".pfnci/wheel-linux/main.sh 12.x 3.7"
}
}
configs {
@@ -56,7 +56,7 @@ configs {
time_limit {
seconds: 10800
}
- command: ".pfnci\\wheel-windows\\main.bat 11.x 3.11"
+ command: ".pfnci\\wheel-windows\\main.bat 12.x 3.11"
}
}
configs {
diff --git a/.pfnci/wheel-windows/_flexci.ps1 b/.pfnci/wheel-windows/_flexci.ps1
index 623be061..67dd95c4 100644
--- a/.pfnci/wheel-windows/_flexci.ps1
+++ b/.pfnci/wheel-windows/_flexci.ps1
@@ -20,19 +20,7 @@ function ActivatePython($version) {
}
function ActivateCUDA($version) {
- if ($version -eq "8.0") {
- $Env:CUDA_PATH = $Env:CUDA_PATH_V8_0
- } elseif ($version -eq "9.0") {
- $Env:CUDA_PATH = $Env:CUDA_PATH_V9_0
- } elseif ($version -eq "9.1") {
- $Env:CUDA_PATH = $Env:CUDA_PATH_V9_1
- } elseif ($version -eq "9.2") {
- $Env:CUDA_PATH = $Env:CUDA_PATH_V9_2
- } elseif ($version -eq "10.0") {
- $Env:CUDA_PATH = $Env:CUDA_PATH_V10_0
- } elseif ($version -eq "10.1") {
- $Env:CUDA_PATH = $Env:CUDA_PATH_V10_1
- } elseif ($version -eq "10.2") {
+ if ($version -eq "10.2") {
$Env:CUDA_PATH = $Env:CUDA_PATH_V10_2
} elseif ($version -eq "11.0") {
$Env:CUDA_PATH = $Env:CUDA_PATH_V11_0
@@ -54,10 +42,43 @@ function ActivateCUDA($version) {
$Env:CUDA_PATH = $Env:CUDA_PATH_V11_8
} elseif ($version -eq "11.x") {
$Env:CUDA_PATH = $Env:CUDA_PATH_V11_8
+ } elseif ($version -eq "12.0") {
+ $Env:CUDA_PATH = $Env:CUDA_PATH_V12_0
+ } elseif ($version -eq "12.x") {
+ $Env:CUDA_PATH = $Env:CUDA_PATH_V12_0
} else {
throw "Unsupported CUDA version: $version"
}
- $Env:PATH = "$Env:CUDA_PATH\bin;$Env:ProgramFiles\NVIDIA Corporation\NvToolsExt\bin\x64;" + $Env:PATH
+ $Env:PATH = "$Env:CUDA_PATH\bin;" + $Env:PATH
+}
+
+function ActivateCuDNN($cudnn_version, $cuda_version) {
+ if ($cudnn_version -eq "8.6") {
+ $cudnn = "v8.6.0"
+ } else {
+ throw "Unsupported cuDNN version: $cudnn_version"
+ }
+
+ if ($cuda_version -eq "10.2") {
+ $cuda = "10"
+ } elseif ($cuda_version.startswith("11.")) {
+ $cuda = "11"
+ } else {
+ throw "Unsupported CUDA version: $cuda_version"
+ }
+
+ $base = "C:\Development\cuDNN\$cudnn\cuda$cuda"
+ $Env:CL = "-I$base\include " + $Env:CL
+ $Env:LINK = "/LIBPATH:$base\lib\x64 " + $Env:LINK
+ $Env:PATH = "$base\bin;" + $Env:PATH
+}
+
+function ActivateNVTX1() {
+ $base = "C:\Development\NvToolsExt"
+ $Env:NVTOOLSEXT_PATH = "C:\Development\NvToolsExt"
+ $Env:CL = "-I$base\include " + $Env:CL
+ $Env:LINK = "/LIBPATH:$base\lib\x64 " + $Env:LINK
+ $Env:PATH = "$base\bin\x64;" + $Env:PATH
}
function IsPullRequestTest() {
diff --git a/.pfnci/wheel-windows/build.ps1 b/.pfnci/wheel-windows/build.ps1
index 68b3a39d..3303d21a 100644
--- a/.pfnci/wheel-windows/build.ps1
+++ b/.pfnci/wheel-windows/build.ps1
@@ -32,16 +32,10 @@ function UninstallCuTENSOR($cuda_path) {
}
}
-# Uninstall existing cuDNN for the default CUDA
-$cuda_path = $Env:CUDA_PATH
-UninstallCuDNN $cuda_path
-
# Activate target CUDA and uninstall existing cuDNN for the target CUDA
ActivateCUDA $cuda
+ActivateNVTX1
$cuda_path = $Env:CUDA_PATH
-UninstallCuDNN $cuda_path
-
-# Note: cuTENSOR is not installed by default, so no need to remove it.
# Activate target Python
ActivatePython $python
@@ -98,7 +92,12 @@ Copy-Item -Path "dll_x64\zlibwapi.dll" -Destination "C:\Windows\System32"
# Verify
echo ">> Starting verification..."
-RunOrDie python ./dist.py --action verify --target wheel-win --python $python --cuda $cuda --dist $wheel_file --test release-tests/common --test release-tests/cudnn --test release-tests/pkg_wheel
+if ($cuda -eq "12.x") {
+ # TODO(kmaehashi): cuDNN for CUDA 12 not available yet
+ RunOrDie python ./dist.py --action verify --target wheel-win --python $python --cuda $cuda --dist $wheel_file --test release-tests/common --test release-tests/pkg_wheel
+} else {
+ RunOrDie python ./dist.py --action verify --target wheel-win --python $python --cuda $cuda --dist $wheel_file --test release-tests/common --test release-tests/cudnn --test release-tests/pkg_wheel
+}
# Show build configuration in CuPy
echo ">> Build configuration"
diff --git a/build.sh b/build.sh
index 95f0bb57..fd88bbed 100755
--- a/build.sh
+++ b/build.sh
@@ -26,6 +26,10 @@ case ${CUDA} in
# CUDA Jetson (wheel)
VERIFY_ARGS="${VERIFY_ARGS} --test release-tests/sparse --test release-tests/cudnn"
;;
+ 12.x )
+ # CUDA 12.0 (wheel) -- cuDNN not available yet
+ VERIFY_ARGS="${VERIFY_ARGS} --test release-tests/sparse --test release-tests/nccl --test release-tests/pkg_wheel"
+ ;;
rocm-* )
# ROCm (wheel)
VERIFY_ARGS="${VERIFY_ARGS}"
diff --git a/builder/base/cuda-runfile/build.sh b/builder/base/cuda-runfile/build.sh
index 56022fe0..d8808b06 100755
--- a/builder/base/cuda-runfile/build.sh
+++ b/builder/base/cuda-runfile/build.sh
@@ -71,6 +71,16 @@ case ${CUDA} in
BASE_IMAGE="oraclelinux:8"
IMAGE_SUFFIX="el8"
;;
+ 12.0 )
+ CUDA_VERSION="12.0.0"
+ CUDA_INSTALLER_URL="https://developer.download.nvidia.com/compute/cuda/12.0.0/local_installers/cuda_12.0.0_525.60.13_linux.run"
+ ;;
+ 12.0-aarch64 )
+ CUDA_VERSION="12.0.0"
+ CUDA_INSTALLER_URL="https://developer.download.nvidia.com/compute/cuda/12.0.0/local_installers/cuda_12.0.0_525.60.13_linux_sbsa.run"
+ BASE_IMAGE="oraclelinux:8"
+ IMAGE_SUFFIX="el8"
+ ;;
* )
echo "Unknown CUDA version: ${CUDA}"
exit 1
diff --git a/check_release_assets.py b/check_release_assets.py
index 87ff95f0..470e17bb 100755
--- a/check_release_assets.py
+++ b/check_release_assets.py
@@ -43,6 +43,7 @@
('cupy-cuda110', _v11_cuda_matrix),
('cupy-cuda111', _v11_cuda_matrix),
('cupy-cuda11x', _v11_cuda_matrix),
+ ('cupy-cuda12x', _v11_cuda_matrix),
('cupy-rocm-4-3', _v11_rocm_matrix),
('cupy-rocm-5-0', _v11_rocm_matrix),
],
@@ -55,6 +56,7 @@
('cupy-cuda110', _v12_cuda_matrix),
('cupy-cuda111', _v12_cuda_matrix),
('cupy-cuda11x', _v12_cuda_matrix + _v12_aarch64_matrix),
+ ('cupy-cuda12x', _v12_cuda_matrix + _v12_aarch64_matrix),
('cupy-rocm-4-3', _v12_rocm_matrix),
('cupy-rocm-5-0', _v12_rocm_matrix),
],
@@ -64,6 +66,7 @@
('cupy-cuda110', _v11_cuda_matrix),
('cupy-cuda111', _v11_cuda_matrix),
('cupy-cuda11x', _v11_cuda_matrix + _v11_aarch64_matrix),
+ ('cupy-cuda12x', _v11_cuda_matrix + _v11_aarch64_matrix),
('cupy-rocm-4-3', _v11_rocm_matrix),
('cupy-rocm-5-0', _v11_rocm_matrix),
],
diff --git a/dist_config.py b/dist_config.py
index 5e1b9cd8..d9050b22 100644
--- a/dist_config.py
+++ b/dist_config.py
@@ -88,116 +88,6 @@
'verify_systems': ['ubuntu18.04'],
'system_packages': '',
},
- '11.2': {
- 'name': 'cupy-cuda112',
- 'kind': 'cuda',
- 'image': 'cupy/cupy-release-tools:cuda-runfile-11.2.0-centos7',
- 'libs': [
- ],
- 'includes': [
- ],
- 'preloads': ['cutensor', 'nccl', 'cudnn'],
- 'verify_image': 'nvidia/cuda:11.2.1-runtime-{system}',
- 'verify_systems': ['ubuntu18.04'],
- 'system_packages': '',
- },
- '11.3': {
- 'name': 'cupy-cuda113',
- 'kind': 'cuda',
- 'image': 'cupy/cupy-release-tools:cuda-runfile-11.3.0-centos7',
- 'libs': [],
- 'includes': [],
- 'preloads': ['cutensor', 'nccl', 'cudnn'],
- 'verify_image': 'nvidia/cuda:11.3.0-runtime-{system}',
- 'verify_systems': ['ubuntu18.04'],
- 'system_packages': '',
- },
- '11.4': {
- 'name': 'cupy-cuda114',
- 'kind': 'cuda',
- 'image': 'cupy/cupy-release-tools:cuda-runfile-11.4.0-centos7',
- 'libs': [],
- 'includes': [],
- 'preloads': ['cutensor', 'nccl', 'cudnn'],
- 'verify_image': 'nvidia/cuda:11.4.0-runtime-{system}',
- 'verify_systems': ['ubuntu18.04'],
- 'system_packages': '',
- },
- '11.5': {
- 'name': 'cupy-cuda115',
- 'kind': 'cuda',
- 'image': 'cupy/cupy-release-tools:cuda-runfile-11.5.0-centos7',
- 'libs': [],
- 'includes': [],
- 'preloads': ['cutensor', 'nccl', 'cudnn'],
- 'verify_image': 'nvidia/cuda:11.5.0-runtime-{system}',
- 'verify_systems': ['ubuntu18.04'],
- 'system_packages': '',
- },
- '11.5-aarch64': {
- 'name': 'cupy-cuda115',
- 'kind': 'cuda',
- 'arch': 'aarch64',
- 'platform_version': '11.5',
- 'image': 'cupy/cupy-release-tools:cuda-runfile-11.5.0-el8-aarch64',
- 'libs': [],
- 'includes': [],
- 'preloads': [],
- 'builder_dockerfile': 'Dockerfile.el8',
- 'verify_image': 'nvidia/cuda:11.5.0-runtime-{system}',
- 'verify_systems': ['ubi8'],
- 'system_packages': '',
- },
- '11.6': {
- 'name': 'cupy-cuda116',
- 'kind': 'cuda',
- 'image': 'cupy/cupy-release-tools:cuda-runfile-11.6.0-centos7',
- 'libs': [],
- 'includes': [],
- 'preloads': ['cutensor', 'nccl', 'cudnn'],
- 'verify_image': 'nvidia/cuda:11.6.0-runtime-{system}',
- 'verify_systems': ['ubuntu18.04'],
- 'system_packages': '',
- },
- '11.6-aarch64': {
- 'name': 'cupy-cuda116',
- 'kind': 'cuda',
- 'arch': 'aarch64',
- 'platform_version': '11.6',
- 'image': 'cupy/cupy-release-tools:cuda-runfile-11.6.0-el8-aarch64',
- 'libs': [],
- 'includes': [],
- 'preloads': [],
- 'builder_dockerfile': 'Dockerfile.el8',
- 'verify_image': 'nvidia/cuda:11.6.0-runtime-{system}',
- 'verify_systems': ['ubi8'],
- 'system_packages': '',
- },
- '11.7': {
- 'name': 'cupy-cuda117',
- 'kind': 'cuda',
- 'image': 'cupy/cupy-release-tools:cuda-runfile-11.7.0-centos7',
- 'libs': [],
- 'includes': [],
- 'preloads': ['cutensor', 'nccl', 'cudnn'],
- 'verify_image': 'nvidia/cuda:11.7.0-runtime-{system}',
- 'verify_systems': ['ubuntu18.04'],
- 'system_packages': '',
- },
- '11.7-aarch64': {
- 'name': 'cupy-cuda117',
- 'kind': 'cuda',
- 'arch': 'aarch64',
- 'platform_version': '11.7',
- 'image': 'cupy/cupy-release-tools:cuda-runfile-11.7.0-el8',
- 'libs': [],
- 'includes': [],
- 'preloads': [],
- 'builder_dockerfile': 'Dockerfile.el8',
- 'verify_image': 'nvidia/cuda:11.7.0-runtime-{system}',
- 'verify_systems': ['ubi8'],
- 'system_packages': '',
- },
'11.x': {
# CUDA Enhanced Compatibility wheel (for CUDA 11.2~11.x)
'name': 'cupy-cuda11x',
@@ -246,17 +136,44 @@
],
'system_packages': '',
},
- 'rocm-4.2': {
- 'name': 'cupy-rocm-4-2',
- 'kind': 'rocm',
- 'platform_version': '4.2',
- 'image': 'rocm/dev-centos-7:4.2',
+ '12.x': {
+ # CUDA Enhanced Compatibility wheel (for CUDA 12.x)
+ 'name': 'cupy-cuda12x',
+ 'kind': 'cuda',
+ 'platform_version': '12.x',
+ # Use the latest CUDA version for build.
+ # Note: CUDA 12 still supports RHEL/CentOS 7 on x86_64
+ 'image': 'cupy/cupy-release-tools:cuda-runfile-12.0.0-centos7',
+ 'libs': [],
+ 'includes': [],
+ # TODO(kmaehashi): cuDNN for CUDA 12 not available yet.
+ # 'preloads': ['cutensor', 'nccl', 'cudnn'],
+ 'preloads': ['cutensor', 'nccl'],
+ 'verify_image': 'nvidia/cuda:{system}',
+ 'verify_systems': [
+ # Test on all supported CUDA version variants.
+ '12.0.0-runtime-ubuntu18.04',
+ ],
+ 'system_packages': '',
+ },
+ '12.x-aarch64': {
+ # CUDA Enhanced Compatibility wheel (for CUDA 12.x)
+ 'name': 'cupy-cuda12x',
+ 'kind': 'cuda',
+ 'arch': 'aarch64',
+ 'platform_version': '12.x',
+ # Use the latest image.
+ 'image': 'cupy/cupy-release-tools:cuda-runfile-12.0.0-el8',
'libs': [],
'includes': [],
'preloads': [],
- 'verify_image': 'rocm/rocm-terminal:4.2',
- 'verify_systems': ['default'],
- 'system_packages': 'rocm-dev hipblas hipfft hipsparse rocsparse rocrand rocthrust rocsolver rocfft hipcub rocprim rccl' # NOQA
+ 'builder_dockerfile': 'Dockerfile.el8',
+ 'verify_image': 'nvidia/cuda:{system}',
+ 'verify_systems': [
+ # Test on all supported CUDA version variants.
+ '12.0.0-runtime-ubi8',
+ ],
+ 'system_packages': '',
},
'rocm-4.3': {
'name': 'cupy-rocm-4-3',
@@ -321,77 +238,29 @@
'cudart_lib': 'cudart64_110', # binary compatible between CUDA 11.x
'check_version': lambda x: 11010 <= x < 11020,
},
- '11.2': {
- 'name': 'cupy-cuda112',
- 'kind': 'cuda',
- 'libs': [
- 'nvToolsExt64_1.dll', # NVIDIA Tools Extension Library
- ],
- 'preloads': ['cutensor', 'cudnn'],
- 'cudart_lib': 'cudart64_110', # binary compatible between CUDA 11.x
- 'check_version': lambda x: 11020 <= x < 11030,
- },
- '11.3': {
- 'name': 'cupy-cuda113',
- 'kind': 'cuda',
- 'libs': [
- 'nvToolsExt64_1.dll', # NVIDIA Tools Extension Library
- ],
- 'preloads': ['cutensor', 'cudnn'],
- 'cudart_lib': 'cudart64_110', # binary compatible between CUDA 11.x
- 'check_version': lambda x: 11030 <= x < 11040,
- },
- '11.4': {
- 'name': 'cupy-cuda114',
- 'kind': 'cuda',
- 'libs': [
- 'nvToolsExt64_1.dll', # NVIDIA Tools Extension Library
- ],
- 'preloads': ['cutensor', 'cudnn'],
- 'cudart_lib': 'cudart64_110', # binary compatible between CUDA 11.x
- 'check_version': lambda x: 11040 <= x < 11050,
- },
- '11.5': {
- 'name': 'cupy-cuda115',
- 'kind': 'cuda',
- 'libs': [
- 'nvToolsExt64_1.dll', # NVIDIA Tools Extension Library
- ],
- 'preloads': ['cutensor', 'cudnn'],
- 'cudart_lib': 'cudart64_110', # binary compatible between CUDA 11.x
- 'check_version': lambda x: 11050 <= x < 11060,
- },
- '11.6': {
- 'name': 'cupy-cuda116',
- 'kind': 'cuda',
- 'libs': [
- 'nvToolsExt64_1.dll', # NVIDIA Tools Extension Library
- ],
- 'preloads': ['cutensor', 'cudnn'],
- 'cudart_lib': 'cudart64_110', # binary compatible between CUDA 11.x
- 'check_version': lambda x: 11060 <= x < 11070,
- },
- '11.7': {
- 'name': 'cupy-cuda117',
+ '11.x': {
+ # CUDA Enhanced Compatibility wheel (for CUDA 11.2~11.x)
+ 'name': 'cupy-cuda11x',
'kind': 'cuda',
'libs': [
'nvToolsExt64_1.dll', # NVIDIA Tools Extension Library
],
'preloads': ['cutensor', 'cudnn'],
'cudart_lib': 'cudart64_110', # binary compatible between CUDA 11.x
- 'check_version': lambda x: 11070 <= x < 11080,
+ 'check_version': lambda x: 11080 <= x < 11090, # CUDA 11.8
},
- '11.x': {
- # CUDA Enhanced Compatibility wheel (for CUDA 11.2~11.x)
- 'name': 'cupy-cuda11x',
+ '12.x': {
+ # CUDA Enhanced Compatibility wheel (for CUDA 12.x)
+ 'name': 'cupy-cuda12x',
'kind': 'cuda',
'libs': [
'nvToolsExt64_1.dll', # NVIDIA Tools Extension Library
],
- 'preloads': ['cutensor', 'cudnn'],
- 'cudart_lib': 'cudart64_110', # binary compatible between CUDA 11.x
- 'check_version': lambda x: 11080 <= x < 11090,
- },
+ # TODO(kmaehashi): cuDNN for CUDA 12 not available yet.
+ 'preloads': ['cutensor'],
+ 'cudart_lib': 'cudart64_12', # binary compatible between CUDA 12.x
+ 'check_version': lambda x: 12000 <= x < 12010, # CUDA 12.0
+ }
}
@@ -412,6 +281,7 @@
This package (``cupy``) is a source distribution.
For most users, use of pre-build wheel distributions are recommended:
+- `cupy-cuda12x `_ (for CUDA 12.x)
- `cupy-cuda11x `_ (for CUDA 11.2 ~ 11.x)
- `cupy-cuda111 `_ (for CUDA 11.1)
- `cupy-cuda110 `_ (for CUDA 11.0)
diff --git a/release-tests/pkg_wheel/test_preload.py b/release-tests/pkg_wheel/test_preload.py
index ad0aa672..70975e88 100644
--- a/release-tests/pkg_wheel/test_preload.py
+++ b/release-tests/pkg_wheel/test_preload.py
@@ -15,6 +15,10 @@ def _get_config(self):
return config
def test_cudnn(self):
+ if cupy.cuda.runtime.runtimeGetVersion() == 12000:
+ # TODO(kmaehashi): cuDNN not yet available for CUDA 12.0
+ assert 'cudnn' not in self._get_config()
+ return
preload_version = self._get_config()['cudnn']['version']
major, minor, patchlevel = (int(x) for x in preload_version.split('.'))
expected_version = major * 1000 + minor * 100 + patchlevel