From 584314d313ca36b42ee8279f9826f6b3b662471a Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 26 Oct 2023 10:05:50 +0200 Subject: [PATCH 1/2] [UR][Tests] Add a CMake option to limit the test devices count and test platforms count in CTS. --- test/conformance/CMakeLists.txt | 4 +++- test/conformance/README.md | 9 ++++++++- test/conformance/cts_exe.py | 6 +++++- test/conformance/source/environment.cpp | 17 +++++++++++++++++ 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/test/conformance/CMakeLists.txt b/test/conformance/CMakeLists.txt index a4c2e8cf94..df80c02681 100644 --- a/test/conformance/CMakeLists.txt +++ b/test/conformance/CMakeLists.txt @@ -4,6 +4,8 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception set(UR_CONFORMANCE_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +option(UR_TEST_DEVICES_COUNT "Count of devices on which CTS will be run" 1) +option(UR_TEST_PLATFORMS_COUNT "Count of platforms on which CTS will be run" 1) function(add_test_adapter name adapter) set(TEST_TARGET_NAME test-${name}) @@ -12,7 +14,7 @@ function(add_test_adapter name adapter) add_test(NAME ${TEST_NAME} COMMAND ${CMAKE_COMMAND} -D TEST_FILE=${Python3_EXECUTABLE} - -D TEST_ARGS="${UR_CONFORMANCE_TEST_DIR}/cts_exe.py --test_command ${CMAKE_BINARY_DIR}/bin/${TEST_TARGET_NAME}" + -D TEST_ARGS="${UR_CONFORMANCE_TEST_DIR}/cts_exe.py --test_command ${CMAKE_BINARY_DIR}/bin/${TEST_TARGET_NAME} --test_devices_count=${UR_TEST_DEVICES_COUNT} --test_platforms_count=${UR_TEST_PLATFORMS_COUNT}" -D MODE=stdout -D MATCH_FILE=${CMAKE_CURRENT_SOURCE_DIR}/${name}_${adapter}.match -P ${PROJECT_SOURCE_DIR}/cmake/match.cmake diff --git a/test/conformance/README.md b/test/conformance/README.md index db90fc759b..9e78337151 100644 --- a/test/conformance/README.md +++ b/test/conformance/README.md @@ -8,4 +8,11 @@ In the future, when all bugs are fixed, and the tests pass, this solution will no longer be necessary. When you fix any test, the match file must be updated Empty match files indicate that there are no failing tests -in a particular group for the corresponding adapter. \ No newline at end of file +in a particular group for the corresponding adapter. + +## How to limit the test devices count + +To limit how many devices you want to run the CTS on, +use CMake option UR_TEST_DEVICES_COUNT. If you want to run +the tests on all available devices, set 0. +The default value is 1. \ No newline at end of file diff --git a/test/conformance/cts_exe.py b/test/conformance/cts_exe.py index ce3ca00a20..55ab134b07 100644 --- a/test/conformance/cts_exe.py +++ b/test/conformance/cts_exe.py @@ -20,9 +20,13 @@ parser = ArgumentParser() parser.add_argument("--test_command", help="Ctest test case") + parser.add_argument("--test_devices_count", type=str, help="Number of devices on which tests will be run") + parser.add_argument("--test_platforms_count", type=str, help="Number of platforms on which tests will be run") args = parser.parse_args() - result = subprocess.Popen([args.test_command, '--gtest_brief=1'], stdout = subprocess.PIPE, text = True) # nosec B603 + result = subprocess.Popen([args.test_command, '--gtest_brief=1', f'--devices_count={args.test_devices_count}', + f'--platforms_count={args.test_platforms_count}'], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) # nosec B603 pat = re.compile(r'\[( )*FAILED( )*\]') output_list = [] diff --git a/test/conformance/source/environment.cpp b/test/conformance/source/environment.cpp index e76b84692c..6918cfb829 100644 --- a/test/conformance/source/environment.cpp +++ b/test/conformance/source/environment.cpp @@ -3,6 +3,7 @@ // See LICENSE.TXT // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +#include #include #include @@ -209,6 +210,22 @@ DevicesEnvironment::DevicesEnvironment(int argc, char **argv) error = "Could not find any devices associated with the platform"; return; } + // Get the argument (test_devices_count) to limit test devices count. + u_long count_set = 0; + for (int i = 1; i < argc; ++i) { + if (std::strcmp(argv[i], "--test_devices_count") == 0 && i + 1 < argc) { + count_set = std::strtoul(argv[i + 1], nullptr, 10); + break; + } + } + // In case, the count_set is "0", the variable count will not be changed. + // The CTS will run on all devices. + if (count_set > (std::numeric_limits::max)()) { + error = "Invalid test_devices_count argument"; + return; + } else if (count_set > 0) { + count = (std::min)(count, static_cast(count_set)); + } devices.resize(count); if (urDeviceGet(platform, UR_DEVICE_TYPE_ALL, count, devices.data(), nullptr)) { From 371bd3cb1d1d9f919557cf3dafc8e5765a88735b Mon Sep 17 00:00:00 2001 From: szadam Date: Wed, 22 Nov 2023 17:30:50 +0100 Subject: [PATCH 2/2] [UR][Tests] Add options in CTS to set test devices count or test device name and test platforms count or test platform name --- test/conformance/README.md | 14 +- test/conformance/source/environment.cpp | 130 ++++++++++++++---- .../testing/include/uur/environment.h | 10 ++ 3 files changed, 125 insertions(+), 29 deletions(-) diff --git a/test/conformance/README.md b/test/conformance/README.md index 9e78337151..e895a5299d 100644 --- a/test/conformance/README.md +++ b/test/conformance/README.md @@ -10,9 +10,13 @@ When you fix any test, the match file must be updated Empty match files indicate that there are no failing tests in a particular group for the corresponding adapter. -## How to limit the test devices count +## How to set test device/platform name or limit the test devices/platforms count -To limit how many devices you want to run the CTS on, -use CMake option UR_TEST_DEVICES_COUNT. If you want to run -the tests on all available devices, set 0. -The default value is 1. \ No newline at end of file +To limit how many devices/platforms you want to run the CTS on, +use CMake option UR_TEST_DEVICES_COUNT or +UR_TEST_PLATFORMS_COUNT. If you want to run the tests on +all available devices/platforms, set 0. The default value is 1. +If you run binaries for the tests, you can use the parameter +`--platforms_count=COUNT` or `--devices_count=COUNT`. +To set test device/platform name you want to run the CTS on, use +parameter `--platform=NAME` or `--device=NAME`. \ No newline at end of file diff --git a/test/conformance/source/environment.cpp b/test/conformance/source/environment.cpp index 6918cfb829..6c917914ed 100644 --- a/test/conformance/source/environment.cpp +++ b/test/conformance/source/environment.cpp @@ -42,6 +42,23 @@ std::ostream &operator<<(std::ostream &out, return out; } +std::ostream &operator<<(std::ostream &out, const ur_device_handle_t &device) { + size_t size; + urDeviceGetInfo(device, UR_DEVICE_INFO_NAME, 0, nullptr, &size); + std::vector name(size); + urDeviceGetInfo(device, UR_DEVICE_INFO_NAME, size, name.data(), nullptr); + out << name.data(); + return out; +} + +std::ostream &operator<<(std::ostream &out, + const std::vector &devices) { + for (auto device : devices) { + out << "\n * \"" << device << "\""; + } + return out; +} + uur::PlatformEnvironment::PlatformEnvironment(int argc, char **argv) : platform_options{parsePlatformOptions(argc, argv)} { instance = this; @@ -101,14 +118,16 @@ uur::PlatformEnvironment::PlatformEnvironment(int argc, char **argv) } if (platform_options.platform_name.empty()) { - if (platforms.size() == 1) { + + if (platforms.size() == 1 || platform_options.platforms_count == 1) { platform = platforms[0]; } else { std::stringstream ss_error; ss_error << "Select a single platform from below using the " "--platform=NAME " "command-line option:" - << platforms; + << platforms << std::endl + << "or set --platforms_count=1."; error = ss_error.str(); return; } @@ -137,7 +156,8 @@ uur::PlatformEnvironment::PlatformEnvironment(int argc, char **argv) << "\" not found. Select a single platform from below " "using the " "--platform=NAME command-line options:" - << platforms; + << platforms << std::endl + << "or set --platforms_count=1."; error = ss_error.str(); return; } @@ -178,6 +198,10 @@ PlatformEnvironment::parsePlatformOptions(int argc, char **argv) { arg, "--platform=", sizeof("--platform=") - 1) == 0) { options.platform_name = std::string(&arg[std::strlen("--platform=")]); + } else if (std::strncmp(arg, "--platforms_count=", + sizeof("--platforms_count=") - 1) == 0) { + options.platforms_count = std::strtoul( + &arg[std::strlen("--platforms_count=")], nullptr, 10); } } @@ -193,10 +217,31 @@ PlatformEnvironment::parsePlatformOptions(int argc, char **argv) { return options; } +DevicesEnvironment::DeviceOptions +DevicesEnvironment::parseDeviceOptions(int argc, char **argv) { + DeviceOptions options; + for (int argi = 1; argi < argc; ++argi) { + const char *arg = argv[argi]; + if (!(std::strcmp(arg, "-h") && std::strcmp(arg, "--help"))) { + // TODO - print help + break; + } else if (std::strncmp(arg, "--device=", sizeof("--device=") - 1) == + 0) { + options.device_name = std::string(&arg[std::strlen("--device=")]); + } else if (std::strncmp(arg, "--devices_count=", + sizeof("--devices_count=") - 1) == 0) { + options.devices_count = std::strtoul( + &arg[std::strlen("--devices_count=")], nullptr, 10); + } + } + return options; +} + DevicesEnvironment *DevicesEnvironment::instance = nullptr; DevicesEnvironment::DevicesEnvironment(int argc, char **argv) - : PlatformEnvironment(argc, argv) { + : PlatformEnvironment(argc, argv), + device_options(parseDeviceOptions(argc, argv)) { instance = this; if (!error.empty()) { return; @@ -210,27 +255,64 @@ DevicesEnvironment::DevicesEnvironment(int argc, char **argv) error = "Could not find any devices associated with the platform"; return; } - // Get the argument (test_devices_count) to limit test devices count. - u_long count_set = 0; - for (int i = 1; i < argc; ++i) { - if (std::strcmp(argv[i], "--test_devices_count") == 0 && i + 1 < argc) { - count_set = std::strtoul(argv[i + 1], nullptr, 10); - break; - } - } - // In case, the count_set is "0", the variable count will not be changed. + + // Get the argument (devices_count) to limit test devices count. + // In case, the devices_count is "0", the variable count will not be changed. // The CTS will run on all devices. - if (count_set > (std::numeric_limits::max)()) { - error = "Invalid test_devices_count argument"; - return; - } else if (count_set > 0) { - count = (std::min)(count, static_cast(count_set)); - } - devices.resize(count); - if (urDeviceGet(platform, UR_DEVICE_TYPE_ALL, count, devices.data(), - nullptr)) { - error = "urDeviceGet() failed to get devices."; - return; + if (device_options.device_name.empty()) { + if (device_options.devices_count > + (std::numeric_limits::max)()) { + error = "Invalid devices_count argument"; + return; + } else if (device_options.devices_count > 0) { + count = (std::min)( + count, static_cast(device_options.devices_count)); + } + devices.resize(count); + if (urDeviceGet(platform, UR_DEVICE_TYPE_ALL, count, devices.data(), + nullptr)) { + error = "urDeviceGet() failed to get devices."; + return; + } + } else { + devices.resize(count); + if (urDeviceGet(platform, UR_DEVICE_TYPE_ALL, count, devices.data(), + nullptr)) { + error = "urDeviceGet() failed to get devices."; + return; + } + for (u_long i = 0; i < count; i++) { + size_t size; + if (urDeviceGetInfo(devices[i], UR_DEVICE_INFO_NAME, 0, nullptr, + &size)) { + error = "urDeviceGetInfo() failed"; + return; + } + std::vector device_name(size); + if (urDeviceGetInfo(devices[i], UR_DEVICE_INFO_NAME, size, + device_name.data(), nullptr)) { + error = "urDeviceGetInfo() failed"; + return; + } + if (device_options.device_name == device_name.data()) { + device = devices[i]; + devices.clear(); + devices.resize(1); + devices[0] = device; + break; + } + } + if (!device) { + std::stringstream ss_error; + ss_error << "Device \"" << device_options.device_name + << "\" not found. Select a single device from below " + "using the " + "--device=NAME command-line options:" + << devices << std::endl + << "or set --devices_count=COUNT."; + error = ss_error.str(); + return; + } } } diff --git a/test/conformance/testing/include/uur/environment.h b/test/conformance/testing/include/uur/environment.h index 5cc6756364..551be76e17 100644 --- a/test/conformance/testing/include/uur/environment.h +++ b/test/conformance/testing/include/uur/environment.h @@ -17,6 +17,7 @@ struct PlatformEnvironment : ::testing::Environment { struct PlatformOptions { std::string platform_name; + unsigned long platforms_count; }; PlatformEnvironment(int argc, char **argv); @@ -36,17 +37,26 @@ struct PlatformEnvironment : ::testing::Environment { struct DevicesEnvironment : PlatformEnvironment { + struct DeviceOptions { + std::string device_name; + unsigned long devices_count; + }; + DevicesEnvironment(int argc, char **argv); virtual ~DevicesEnvironment() override = default; virtual void SetUp() override; virtual void TearDown() override; + DeviceOptions parseDeviceOptions(int argc, char **argv); + inline const std::vector &GetDevices() const { return devices; } + DeviceOptions device_options; std::vector devices; + ur_device_handle_t device = nullptr; static DevicesEnvironment *instance; };