Merge pull request #1085 from szadam/limit_devices_count

[UR][Tests] Add CTS options to set platforms and devices.

test/conformance/CMakeLists.txt

@@ -4,6 +4,8 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 set(UR_CONFORMANCE_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR})
+option(UR_TEST_DEVICES_COUNT "Count of devices on which CTS will be run" 1)
+option(UR_TEST_PLATFORMS_COUNT "Count of platforms on which CTS will be run" 1)
 
 function(add_test_adapter name adapter)
     set(TEST_TARGET_NAME test-${name})
@@ -12,7 +14,7 @@ function(add_test_adapter name adapter)
     add_test(NAME ${TEST_NAME}
         COMMAND ${CMAKE_COMMAND}
         -D TEST_FILE=${Python3_EXECUTABLE}
-        -D TEST_ARGS="${UR_CONFORMANCE_TEST_DIR}/cts_exe.py --test_command ${CMAKE_BINARY_DIR}/bin/${TEST_TARGET_NAME}"
+        -D TEST_ARGS="${UR_CONFORMANCE_TEST_DIR}/cts_exe.py --test_command ${CMAKE_BINARY_DIR}/bin/${TEST_TARGET_NAME} --test_devices_count=${UR_TEST_DEVICES_COUNT} --test_platforms_count=${UR_TEST_PLATFORMS_COUNT}"
         -D MODE=stdout
         -D MATCH_FILE=${CMAKE_CURRENT_SOURCE_DIR}/${name}_${adapter}.match
         -P ${PROJECT_SOURCE_DIR}/cmake/match.cmake

test/conformance/README.md

@@ -8,4 +8,15 @@ In the future, when all bugs are fixed, and the tests pass,
 this solution will no longer be necessary.
 When you fix any test, the match file must be updated
 Empty match files indicate that there are no failing tests
-in a particular group for the corresponding adapter.
+in a particular group for the corresponding adapter.
+
+## How to set test device/platform name or limit the test devices/platforms count
+
+To limit how many devices/platforms you want to run the CTS on,
+use CMake option UR_TEST_DEVICES_COUNT or
+UR_TEST_PLATFORMS_COUNT. If you want to run the tests on
+all available devices/platforms, set 0. The default value is 1.
+If you run binaries for the tests, you can use the parameter
+`--platforms_count=COUNT` or `--devices_count=COUNT`.
+To set test device/platform name you want to run the CTS on, use
+parameter `--platform=NAME` or `--device=NAME`.

test/conformance/cts_exe.py

@@ -20,9 +20,13 @@
 
     parser = ArgumentParser()
     parser.add_argument("--test_command", help="Ctest test case")
+    parser.add_argument("--test_devices_count", type=str, help="Number of devices on which tests will be run")
+    parser.add_argument("--test_platforms_count", type=str, help="Number of platforms on which tests will be run")
     args = parser.parse_args()
 
-    result = subprocess.Popen([args.test_command, '--gtest_brief=1'], stdout = subprocess.PIPE, text = True)  # nosec B603
+    result = subprocess.Popen([args.test_command, '--gtest_brief=1', f'--devices_count={args.test_devices_count}',
+                               f'--platforms_count={args.test_platforms_count}'],
+                               stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)  # nosec B603
 
     pat = re.compile(r'\[( )*FAILED( )*\]')
     output_list = []

test/conformance/source/environment.cpp

@@ -3,6 +3,7 @@
 // See LICENSE.TXT
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+#include <algorithm>
 #include <cstring>
 #include <fstream>
 
@@ -41,6 +42,23 @@ std::ostream &operator<<(std::ostream &out,
     return out;
 }
 
+std::ostream &operator<<(std::ostream &out, const ur_device_handle_t &device) {
+    size_t size;
+    urDeviceGetInfo(device, UR_DEVICE_INFO_NAME, 0, nullptr, &size);
+    std::vector<char> name(size);
+    urDeviceGetInfo(device, UR_DEVICE_INFO_NAME, size, name.data(), nullptr);
+    out << name.data();
+    return out;
+}
+
+std::ostream &operator<<(std::ostream &out,
+                         const std::vector<ur_device_handle_t> &devices) {
+    for (auto device : devices) {
+        out << "\n  * \"" << device << "\"";
+    }
+    return out;
+}
+
 uur::PlatformEnvironment::PlatformEnvironment(int argc, char **argv)
     : platform_options{parsePlatformOptions(argc, argv)} {
     instance = this;
@@ -100,14 +118,16 @@ uur::PlatformEnvironment::PlatformEnvironment(int argc, char **argv)
     }
 
     if (platform_options.platform_name.empty()) {
-        if (platforms.size() == 1) {
+
+        if (platforms.size() == 1 || platform_options.platforms_count == 1) {
             platform = platforms[0];
         } else {
             std::stringstream ss_error;
             ss_error << "Select a single platform from below using the "
                         "--platform=NAME "
                         "command-line option:"
-                     << platforms;
+                     << platforms << std::endl
+                     << "or set --platforms_count=1.";
             error = ss_error.str();
             return;
         }
@@ -136,7 +156,8 @@ uur::PlatformEnvironment::PlatformEnvironment(int argc, char **argv)
                      << "\" not found. Select a single platform from below "
                         "using the "
                         "--platform=NAME command-line options:"
-                     << platforms;
+                     << platforms << std::endl
+                     << "or set --platforms_count=1.";
             error = ss_error.str();
             return;
         }
@@ -177,6 +198,10 @@ PlatformEnvironment::parsePlatformOptions(int argc, char **argv) {
                        arg, "--platform=", sizeof("--platform=") - 1) == 0) {
             options.platform_name =
                 std::string(&arg[std::strlen("--platform=")]);
+        } else if (std::strncmp(arg, "--platforms_count=",
+                                sizeof("--platforms_count=") - 1) == 0) {
+            options.platforms_count = std::strtoul(
+                &arg[std::strlen("--platforms_count=")], nullptr, 10);
         }
     }
 
@@ -192,10 +217,31 @@ PlatformEnvironment::parsePlatformOptions(int argc, char **argv) {
     return options;
 }
 
+DevicesEnvironment::DeviceOptions
+DevicesEnvironment::parseDeviceOptions(int argc, char **argv) {
+    DeviceOptions options;
+    for (int argi = 1; argi < argc; ++argi) {
+        const char *arg = argv[argi];
+        if (!(std::strcmp(arg, "-h") && std::strcmp(arg, "--help"))) {
+            // TODO - print help
+            break;
+        } else if (std::strncmp(arg, "--device=", sizeof("--device=") - 1) ==
+                   0) {
+            options.device_name = std::string(&arg[std::strlen("--device=")]);
+        } else if (std::strncmp(arg, "--devices_count=",
+                                sizeof("--devices_count=") - 1) == 0) {
+            options.devices_count = std::strtoul(
+                &arg[std::strlen("--devices_count=")], nullptr, 10);
+        }
+    }
+    return options;
+}
+
 DevicesEnvironment *DevicesEnvironment::instance = nullptr;
 
 DevicesEnvironment::DevicesEnvironment(int argc, char **argv)
-    : PlatformEnvironment(argc, argv) {
+    : PlatformEnvironment(argc, argv),
+      device_options(parseDeviceOptions(argc, argv)) {
     instance = this;
     if (!error.empty()) {
         return;
@@ -209,11 +255,64 @@ DevicesEnvironment::DevicesEnvironment(int argc, char **argv)
         error = "Could not find any devices associated with the platform";
         return;
     }
-    devices.resize(count);
-    if (urDeviceGet(platform, UR_DEVICE_TYPE_ALL, count, devices.data(),
-                    nullptr)) {
-        error = "urDeviceGet() failed to get devices.";
-        return;
+
+    // Get the argument (devices_count) to limit test devices count.
+    // In case, the devices_count is "0", the variable count will not be changed.
+    // The CTS will run on all devices.
+    if (device_options.device_name.empty()) {
+        if (device_options.devices_count >
+            (std::numeric_limits<uint32_t>::max)()) {
+            error = "Invalid devices_count argument";
+            return;
+        } else if (device_options.devices_count > 0) {
+            count = (std::min)(
+                count, static_cast<uint32_t>(device_options.devices_count));
+        }
+        devices.resize(count);
+        if (urDeviceGet(platform, UR_DEVICE_TYPE_ALL, count, devices.data(),
+                        nullptr)) {
+            error = "urDeviceGet() failed to get devices.";
+            return;
+        }
+    } else {
+        devices.resize(count);
+        if (urDeviceGet(platform, UR_DEVICE_TYPE_ALL, count, devices.data(),
+                        nullptr)) {
+            error = "urDeviceGet() failed to get devices.";
+            return;
+        }
+        for (u_long i = 0; i < count; i++) {
+            size_t size;
+            if (urDeviceGetInfo(devices[i], UR_DEVICE_INFO_NAME, 0, nullptr,
+                                &size)) {
+                error = "urDeviceGetInfo() failed";
+                return;
+            }
+            std::vector<char> device_name(size);
+            if (urDeviceGetInfo(devices[i], UR_DEVICE_INFO_NAME, size,
+                                device_name.data(), nullptr)) {
+                error = "urDeviceGetInfo() failed";
+                return;
+            }
+            if (device_options.device_name == device_name.data()) {
+                device = devices[i];
+                devices.clear();
+                devices.resize(1);
+                devices[0] = device;
+                break;
+            }
+        }
+        if (!device) {
+            std::stringstream ss_error;
+            ss_error << "Device \"" << device_options.device_name
+                     << "\" not found. Select a single device from below "
+                        "using the "
+                        "--device=NAME command-line options:"
+                     << devices << std::endl
+                     << "or set --devices_count=COUNT.";
+            error = ss_error.str();
+            return;
+        }
     }
 }
 

test/conformance/testing/include/uur/environment.h

@@ -17,6 +17,7 @@ struct PlatformEnvironment : ::testing::Environment {
 
     struct PlatformOptions {
         std::string platform_name;
+        unsigned long platforms_count;
     };
 
     PlatformEnvironment(int argc, char **argv);
@@ -36,17 +37,26 @@ struct PlatformEnvironment : ::testing::Environment {
 
 struct DevicesEnvironment : PlatformEnvironment {
 
+    struct DeviceOptions {
+        std::string device_name;
+        unsigned long devices_count;
+    };
+
     DevicesEnvironment(int argc, char **argv);
     virtual ~DevicesEnvironment() override = default;
 
     virtual void SetUp() override;
     virtual void TearDown() override;
 
+    DeviceOptions parseDeviceOptions(int argc, char **argv);
+
     inline const std::vector<ur_device_handle_t> &GetDevices() const {
         return devices;
     }
 
+    DeviceOptions device_options;
     std::vector<ur_device_handle_t> devices;
+    ur_device_handle_t device = nullptr;
     static DevicesEnvironment *instance;
 };