diff --git a/test/conformance/CMakeLists.txt b/test/conformance/CMakeLists.txt index a4c2e8cf94..df80c02681 100644 --- a/test/conformance/CMakeLists.txt +++ b/test/conformance/CMakeLists.txt @@ -4,6 +4,8 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception set(UR_CONFORMANCE_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +option(UR_TEST_DEVICES_COUNT "Count of devices on which CTS will be run" 1) +option(UR_TEST_PLATFORMS_COUNT "Count of platforms on which CTS will be run" 1) function(add_test_adapter name adapter) set(TEST_TARGET_NAME test-${name}) @@ -12,7 +14,7 @@ function(add_test_adapter name adapter) add_test(NAME ${TEST_NAME} COMMAND ${CMAKE_COMMAND} -D TEST_FILE=${Python3_EXECUTABLE} - -D TEST_ARGS="${UR_CONFORMANCE_TEST_DIR}/cts_exe.py --test_command ${CMAKE_BINARY_DIR}/bin/${TEST_TARGET_NAME}" + -D TEST_ARGS="${UR_CONFORMANCE_TEST_DIR}/cts_exe.py --test_command ${CMAKE_BINARY_DIR}/bin/${TEST_TARGET_NAME} --test_devices_count=${UR_TEST_DEVICES_COUNT} --test_platforms_count=${UR_TEST_PLATFORMS_COUNT}" -D MODE=stdout -D MATCH_FILE=${CMAKE_CURRENT_SOURCE_DIR}/${name}_${adapter}.match -P ${PROJECT_SOURCE_DIR}/cmake/match.cmake diff --git a/test/conformance/README.md b/test/conformance/README.md index db90fc759b..e895a5299d 100644 --- a/test/conformance/README.md +++ b/test/conformance/README.md @@ -8,4 +8,15 @@ In the future, when all bugs are fixed, and the tests pass, this solution will no longer be necessary. When you fix any test, the match file must be updated Empty match files indicate that there are no failing tests -in a particular group for the corresponding adapter. \ No newline at end of file +in a particular group for the corresponding adapter. + +## How to set test device/platform name or limit the test devices/platforms count + +To limit how many devices/platforms you want to run the CTS on, +use CMake option UR_TEST_DEVICES_COUNT or +UR_TEST_PLATFORMS_COUNT. If you want to run the tests on +all available devices/platforms, set 0. The default value is 1. +If you run binaries for the tests, you can use the parameter +`--platforms_count=COUNT` or `--devices_count=COUNT`. +To set test device/platform name you want to run the CTS on, use +parameter `--platform=NAME` or `--device=NAME`. \ No newline at end of file diff --git a/test/conformance/cts_exe.py b/test/conformance/cts_exe.py index ce3ca00a20..55ab134b07 100644 --- a/test/conformance/cts_exe.py +++ b/test/conformance/cts_exe.py @@ -20,9 +20,13 @@ parser = ArgumentParser() parser.add_argument("--test_command", help="Ctest test case") + parser.add_argument("--test_devices_count", type=str, help="Number of devices on which tests will be run") + parser.add_argument("--test_platforms_count", type=str, help="Number of platforms on which tests will be run") args = parser.parse_args() - result = subprocess.Popen([args.test_command, '--gtest_brief=1'], stdout = subprocess.PIPE, text = True) # nosec B603 + result = subprocess.Popen([args.test_command, '--gtest_brief=1', f'--devices_count={args.test_devices_count}', + f'--platforms_count={args.test_platforms_count}'], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) # nosec B603 pat = re.compile(r'\[( )*FAILED( )*\]') output_list = [] diff --git a/test/conformance/source/environment.cpp b/test/conformance/source/environment.cpp index e76b84692c..6c917914ed 100644 --- a/test/conformance/source/environment.cpp +++ b/test/conformance/source/environment.cpp @@ -3,6 +3,7 @@ // See LICENSE.TXT // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +#include #include #include @@ -41,6 +42,23 @@ std::ostream &operator<<(std::ostream &out, return out; } +std::ostream &operator<<(std::ostream &out, const ur_device_handle_t &device) { + size_t size; + urDeviceGetInfo(device, UR_DEVICE_INFO_NAME, 0, nullptr, &size); + std::vector name(size); + urDeviceGetInfo(device, UR_DEVICE_INFO_NAME, size, name.data(), nullptr); + out << name.data(); + return out; +} + +std::ostream &operator<<(std::ostream &out, + const std::vector &devices) { + for (auto device : devices) { + out << "\n * \"" << device << "\""; + } + return out; +} + uur::PlatformEnvironment::PlatformEnvironment(int argc, char **argv) : platform_options{parsePlatformOptions(argc, argv)} { instance = this; @@ -100,14 +118,16 @@ uur::PlatformEnvironment::PlatformEnvironment(int argc, char **argv) } if (platform_options.platform_name.empty()) { - if (platforms.size() == 1) { + + if (platforms.size() == 1 || platform_options.platforms_count == 1) { platform = platforms[0]; } else { std::stringstream ss_error; ss_error << "Select a single platform from below using the " "--platform=NAME " "command-line option:" - << platforms; + << platforms << std::endl + << "or set --platforms_count=1."; error = ss_error.str(); return; } @@ -136,7 +156,8 @@ uur::PlatformEnvironment::PlatformEnvironment(int argc, char **argv) << "\" not found. Select a single platform from below " "using the " "--platform=NAME command-line options:" - << platforms; + << platforms << std::endl + << "or set --platforms_count=1."; error = ss_error.str(); return; } @@ -177,6 +198,10 @@ PlatformEnvironment::parsePlatformOptions(int argc, char **argv) { arg, "--platform=", sizeof("--platform=") - 1) == 0) { options.platform_name = std::string(&arg[std::strlen("--platform=")]); + } else if (std::strncmp(arg, "--platforms_count=", + sizeof("--platforms_count=") - 1) == 0) { + options.platforms_count = std::strtoul( + &arg[std::strlen("--platforms_count=")], nullptr, 10); } } @@ -192,10 +217,31 @@ PlatformEnvironment::parsePlatformOptions(int argc, char **argv) { return options; } +DevicesEnvironment::DeviceOptions +DevicesEnvironment::parseDeviceOptions(int argc, char **argv) { + DeviceOptions options; + for (int argi = 1; argi < argc; ++argi) { + const char *arg = argv[argi]; + if (!(std::strcmp(arg, "-h") && std::strcmp(arg, "--help"))) { + // TODO - print help + break; + } else if (std::strncmp(arg, "--device=", sizeof("--device=") - 1) == + 0) { + options.device_name = std::string(&arg[std::strlen("--device=")]); + } else if (std::strncmp(arg, "--devices_count=", + sizeof("--devices_count=") - 1) == 0) { + options.devices_count = std::strtoul( + &arg[std::strlen("--devices_count=")], nullptr, 10); + } + } + return options; +} + DevicesEnvironment *DevicesEnvironment::instance = nullptr; DevicesEnvironment::DevicesEnvironment(int argc, char **argv) - : PlatformEnvironment(argc, argv) { + : PlatformEnvironment(argc, argv), + device_options(parseDeviceOptions(argc, argv)) { instance = this; if (!error.empty()) { return; @@ -209,11 +255,64 @@ DevicesEnvironment::DevicesEnvironment(int argc, char **argv) error = "Could not find any devices associated with the platform"; return; } - devices.resize(count); - if (urDeviceGet(platform, UR_DEVICE_TYPE_ALL, count, devices.data(), - nullptr)) { - error = "urDeviceGet() failed to get devices."; - return; + + // Get the argument (devices_count) to limit test devices count. + // In case, the devices_count is "0", the variable count will not be changed. + // The CTS will run on all devices. + if (device_options.device_name.empty()) { + if (device_options.devices_count > + (std::numeric_limits::max)()) { + error = "Invalid devices_count argument"; + return; + } else if (device_options.devices_count > 0) { + count = (std::min)( + count, static_cast(device_options.devices_count)); + } + devices.resize(count); + if (urDeviceGet(platform, UR_DEVICE_TYPE_ALL, count, devices.data(), + nullptr)) { + error = "urDeviceGet() failed to get devices."; + return; + } + } else { + devices.resize(count); + if (urDeviceGet(platform, UR_DEVICE_TYPE_ALL, count, devices.data(), + nullptr)) { + error = "urDeviceGet() failed to get devices."; + return; + } + for (u_long i = 0; i < count; i++) { + size_t size; + if (urDeviceGetInfo(devices[i], UR_DEVICE_INFO_NAME, 0, nullptr, + &size)) { + error = "urDeviceGetInfo() failed"; + return; + } + std::vector device_name(size); + if (urDeviceGetInfo(devices[i], UR_DEVICE_INFO_NAME, size, + device_name.data(), nullptr)) { + error = "urDeviceGetInfo() failed"; + return; + } + if (device_options.device_name == device_name.data()) { + device = devices[i]; + devices.clear(); + devices.resize(1); + devices[0] = device; + break; + } + } + if (!device) { + std::stringstream ss_error; + ss_error << "Device \"" << device_options.device_name + << "\" not found. Select a single device from below " + "using the " + "--device=NAME command-line options:" + << devices << std::endl + << "or set --devices_count=COUNT."; + error = ss_error.str(); + return; + } } } diff --git a/test/conformance/testing/include/uur/environment.h b/test/conformance/testing/include/uur/environment.h index 5cc6756364..551be76e17 100644 --- a/test/conformance/testing/include/uur/environment.h +++ b/test/conformance/testing/include/uur/environment.h @@ -17,6 +17,7 @@ struct PlatformEnvironment : ::testing::Environment { struct PlatformOptions { std::string platform_name; + unsigned long platforms_count; }; PlatformEnvironment(int argc, char **argv); @@ -36,17 +37,26 @@ struct PlatformEnvironment : ::testing::Environment { struct DevicesEnvironment : PlatformEnvironment { + struct DeviceOptions { + std::string device_name; + unsigned long devices_count; + }; + DevicesEnvironment(int argc, char **argv); virtual ~DevicesEnvironment() override = default; virtual void SetUp() override; virtual void TearDown() override; + DeviceOptions parseDeviceOptions(int argc, char **argv); + inline const std::vector &GetDevices() const { return devices; } + DeviceOptions device_options; std::vector devices; + ur_device_handle_t device = nullptr; static DevicesEnvironment *instance; };