From 73a405b58170e314c3a621655025411dc4a6f4ef Mon Sep 17 00:00:00 2001 From: "Kenneth Benzie (Benie)" Date: Mon, 26 Feb 2024 11:07:36 +0000 Subject: [PATCH 01/11] [Doc] Define the etor/etors abbreviations Fixes #1381 by adding a paragraph defining that etor/etors are abbreviations for enumerator/enumerators. --- scripts/YaML.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/YaML.md b/scripts/YaML.md index ee22cd39d6..a995b5858c 100644 --- a/scripts/YaML.md +++ b/scripts/YaML.md @@ -314,6 +314,10 @@ class ur_name_handle_t(c_void_p): #### type: enum + +In the following section the word *enumerator* is abbreviated to `etor` and the +plural form *enumerators* is abbreviated to `etors`. + * An enum initiates the creation of a C/C++ `enum` declaration in the specification * An enum requires the following scalar fields: {`desc`, `name`} - `desc` will be used as the enum's description comment From 8fa74e28e66c99b5e1bb6708d4e5890ae3303e0c Mon Sep 17 00:00:00 2001 From: "He, Wenju" Date: Mon, 26 Feb 2024 21:49:24 -0800 Subject: [PATCH 02/11] [UR][L0] Implement bindless image functions from ZE_experimental_bindless_image Function zeMemGetPitchFor2dImage and zeImageGetDeviceOffsetExp from L0 extension ZE_experimental_bindless_image are used. In urBindlessImagesUnsampledImageCreateExp, if image description is different from that of image allocated by urBindlessImagesImageAllocateExp, an image view is created from the image. --- source/adapters/level_zero/CMakeLists.txt | 2 +- source/adapters/level_zero/device.cpp | 25 + source/adapters/level_zero/image.cpp | 913 ++++++++++++++++++++-- source/adapters/level_zero/image.hpp | 8 + source/adapters/level_zero/memory.cpp | 126 +-- source/adapters/level_zero/memory.hpp | 4 +- 6 files changed, 892 insertions(+), 186 deletions(-) diff --git a/source/adapters/level_zero/CMakeLists.txt b/source/adapters/level_zero/CMakeLists.txt index adde1ec7d6..62ba260dea 100644 --- a/source/adapters/level_zero/CMakeLists.txt +++ b/source/adapters/level_zero/CMakeLists.txt @@ -34,7 +34,7 @@ if (NOT DEFINED LEVEL_ZERO_LIBRARY OR NOT DEFINED LEVEL_ZERO_INCLUDE_DIR) endif() set(LEVEL_ZERO_LOADER_REPO "https://github.com/oneapi-src/level-zero.git") - set(LEVEL_ZERO_LOADER_TAG v1.15.1) + set(LEVEL_ZERO_LOADER_TAG v1.16.1) # Disable due to a bug https://github.com/oneapi-src/level-zero/issues/104 set(CMAKE_INCLUDE_CURRENT_DIR OFF) diff --git a/source/adapters/level_zero/device.cpp b/source/adapters/level_zero/device.cpp index 918b04400a..88916329ad 100644 --- a/source/adapters/level_zero/device.cpp +++ b/source/adapters/level_zero/device.cpp @@ -919,6 +919,31 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo( return ReturnValue(true); case UR_DEVICE_INFO_COMMAND_BUFFER_UPDATE_SUPPORT_EXP: return ReturnValue(false); + case UR_DEVICE_INFO_BINDLESS_IMAGES_SUPPORT_EXP: + return ReturnValue(true); + case UR_DEVICE_INFO_BINDLESS_IMAGES_SHARED_USM_SUPPORT_EXP: + return ReturnValue(true); + case UR_DEVICE_INFO_BINDLESS_IMAGES_1D_USM_SUPPORT_EXP: + return ReturnValue(true); + case UR_DEVICE_INFO_BINDLESS_IMAGES_2D_USM_SUPPORT_EXP: + return ReturnValue(true); + case UR_DEVICE_INFO_IMAGE_PITCH_ALIGN_EXP: + case UR_DEVICE_INFO_MAX_IMAGE_LINEAR_WIDTH_EXP: + case UR_DEVICE_INFO_MAX_IMAGE_LINEAR_HEIGHT_EXP: + case UR_DEVICE_INFO_MAX_IMAGE_LINEAR_PITCH_EXP: + urPrint("Unsupported ParamName in urGetDeviceInfo\n"); + urPrint("ParamName=%d(0x%x)\n", ParamName, ParamName); + return UR_RESULT_ERROR_INVALID_VALUE; + case UR_DEVICE_INFO_MIPMAP_SUPPORT_EXP: + return ReturnValue(true); + case UR_DEVICE_INFO_MIPMAP_ANISOTROPY_SUPPORT_EXP: + return ReturnValue(true); + case UR_DEVICE_INFO_MIPMAP_MAX_ANISOTROPY_EXP: + case UR_DEVICE_INFO_MIPMAP_LEVEL_REFERENCE_SUPPORT_EXP: + case UR_DEVICE_INFO_INTEROP_MEMORY_IMPORT_SUPPORT_EXP: + case UR_DEVICE_INFO_INTEROP_MEMORY_EXPORT_SUPPORT_EXP: + case UR_DEVICE_INFO_INTEROP_SEMAPHORE_IMPORT_SUPPORT_EXP: + case UR_DEVICE_INFO_INTEROP_SEMAPHORE_EXPORT_SUPPORT_EXP: default: urPrint("Unsupported ParamName in urGetDeviceInfo\n"); urPrint("ParamName=%d(0x%x)\n", ParamName, ParamName); diff --git a/source/adapters/level_zero/image.cpp b/source/adapters/level_zero/image.cpp index 3e2b78488f..6ff38672b7 100644 --- a/source/adapters/level_zero/image.cpp +++ b/source/adapters/level_zero/image.cpp @@ -10,23 +10,582 @@ #include "image.hpp" #include "common.hpp" +#include "context.hpp" +#include "event.hpp" +#include "sampler.hpp" +#include "ur_level_zero.hpp" + +typedef ze_result_t(ZE_APICALL *zeImageGetDeviceOffsetExp_pfn)( + ze_image_handle_t hImage, uint64_t *pDeviceOffset); + +typedef ze_result_t(ZE_APICALL *zeMemGetPitchFor2dImage_pfn)( + ze_context_handle_t hContext, ze_device_handle_t hDevice, size_t imageWidth, + size_t imageHeight, unsigned int elementSizeInBytes, size_t *rowPitch); + +namespace { + +zeMemGetPitchFor2dImage_pfn zeMemGetPitchFor2dImageFunctionPtr = nullptr; + +zeImageGetDeviceOffsetExp_pfn zeImageGetDeviceOffsetExpFunctionPtr = nullptr; + +/// Return true if the two image_desc are the same. +bool isSameImageDesc(const ze_image_desc_t *Desc1, + const ze_image_desc_t *Desc2) { + auto IsSameImageFormat = [](const ze_image_format_t &Format1, + const ze_image_format_t &Format2) { + return Format1.layout == Format2.layout && Format1.type == Format2.type && + Format1.x == Format2.x && Format1.y == Format2.y && + Format1.z == Format2.z && Format1.w == Format2.w; + }; + return Desc1->stype == Desc2->stype && Desc1->flags == Desc2->flags && + Desc1->type == Desc2->type && + IsSameImageFormat(Desc1->format, Desc2->format) && + Desc1->width == Desc2->width && Desc1->height == Desc2->height && + Desc1->depth == Desc2->depth && + Desc1->arraylevels == Desc2->arraylevels && + Desc1->miplevels == Desc2->miplevels; +} + +/// Construct UR image format from ZE image desc. +ur_result_t ze2urImageFormat(const ze_image_desc_t *ZeImageDesc, + ur_image_format_t *UrImageFormat) { + const ze_image_format_t &ZeImageFormat = ZeImageDesc->format; + size_t ZeImageFormatTypeSize; + switch (ZeImageFormat.layout) { + case ZE_IMAGE_FORMAT_LAYOUT_8: + case ZE_IMAGE_FORMAT_LAYOUT_8_8: + case ZE_IMAGE_FORMAT_LAYOUT_8_8_8_8: + ZeImageFormatTypeSize = 8; + break; + case ZE_IMAGE_FORMAT_LAYOUT_16: + case ZE_IMAGE_FORMAT_LAYOUT_16_16: + case ZE_IMAGE_FORMAT_LAYOUT_16_16_16_16: + ZeImageFormatTypeSize = 16; + break; + case ZE_IMAGE_FORMAT_LAYOUT_32: + case ZE_IMAGE_FORMAT_LAYOUT_32_32: + case ZE_IMAGE_FORMAT_LAYOUT_32_32_32_32: + ZeImageFormatTypeSize = 32; + break; + default: + urPrint("ze2urImageFormat: unsupported image format layout: layout = %d\n", + ZeImageFormat.layout); + return UR_RESULT_ERROR_INVALID_VALUE; + } + + ur_image_channel_order_t ChannelOrder; + switch (ZeImageFormat.layout) { + case ZE_IMAGE_FORMAT_LAYOUT_8: + case ZE_IMAGE_FORMAT_LAYOUT_16: + case ZE_IMAGE_FORMAT_LAYOUT_32: + switch (ZeImageFormat.x) { + case ZE_IMAGE_FORMAT_SWIZZLE_R: + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_R; + break; + case ZE_IMAGE_FORMAT_SWIZZLE_A: + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_A; + break; + default: + urPrint("ze2urImageFormat: unexpected image format channel x: x = %d\n", + ZeImageFormat.x); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + case ZE_IMAGE_FORMAT_LAYOUT_8_8: + case ZE_IMAGE_FORMAT_LAYOUT_16_16: + case ZE_IMAGE_FORMAT_LAYOUT_32_32: + if (ZeImageFormat.x != ZE_IMAGE_FORMAT_SWIZZLE_R) { + urPrint("ze2urImageFormat: unexpected image format channel x: x = %d\n", + ZeImageFormat.x); + return UR_RESULT_ERROR_INVALID_VALUE; + } + switch (ZeImageFormat.y) { + case ZE_IMAGE_FORMAT_SWIZZLE_G: + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RG; + break; + case ZE_IMAGE_FORMAT_SWIZZLE_A: + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RA; + break; + case ZE_IMAGE_FORMAT_SWIZZLE_X: + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RX; + break; + default: + urPrint("ze2urImageFormat: unexpected image format channel y: y = %d\n", + ZeImageFormat.x); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + case ZE_IMAGE_FORMAT_LAYOUT_8_8_8_8: + case ZE_IMAGE_FORMAT_LAYOUT_16_16_16_16: + case ZE_IMAGE_FORMAT_LAYOUT_32_32_32_32: + if (ZeImageFormat.x == ZE_IMAGE_FORMAT_SWIZZLE_R && + ZeImageFormat.y == ZE_IMAGE_FORMAT_SWIZZLE_G && + ZeImageFormat.z == ZE_IMAGE_FORMAT_SWIZZLE_B) { + switch (ZeImageFormat.w) { + case ZE_IMAGE_FORMAT_SWIZZLE_X: + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RGBX; + break; + case ZE_IMAGE_FORMAT_SWIZZLE_A: + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RGBA; + break; + default: + urPrint("ze2urImageFormat: unexpected image format channel w: w = %d\n", + ZeImageFormat.x); + return UR_RESULT_ERROR_INVALID_VALUE; + } + } else if (ZeImageFormat.x == ZE_IMAGE_FORMAT_SWIZZLE_A && + ZeImageFormat.y == ZE_IMAGE_FORMAT_SWIZZLE_R && + ZeImageFormat.z == ZE_IMAGE_FORMAT_SWIZZLE_G && + ZeImageFormat.w == ZE_IMAGE_FORMAT_SWIZZLE_B) { + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_ARGB; + } else if (ZeImageFormat.x == ZE_IMAGE_FORMAT_SWIZZLE_B && + ZeImageFormat.y == ZE_IMAGE_FORMAT_SWIZZLE_G && + ZeImageFormat.z == ZE_IMAGE_FORMAT_SWIZZLE_R && + ZeImageFormat.w == ZE_IMAGE_FORMAT_SWIZZLE_A) { + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_BGRA; + } else { + urPrint("ze2urImageFormat: unexpected image format channel\n"); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + default: + urPrint("ze2urImageFormat: unsupported image format layout: layout = %d\n", + ZeImageFormat.layout); + return UR_RESULT_ERROR_INVALID_VALUE; + } + + ur_image_channel_type_t ChannelType; + switch (ZeImageFormat.type) { + case ZE_IMAGE_FORMAT_TYPE_UINT: + switch (ZeImageFormatTypeSize) { + case 8: + ChannelType = UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT8; + break; + case 16: + ChannelType = UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT16; + break; + case 32: + ChannelType = UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT32; + break; + default: + urPrint( + "ze2urImageFormat: unexpected image format type size: size = %zu\n", + ZeImageFormatTypeSize); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + case ZE_IMAGE_FORMAT_TYPE_SINT: + switch (ZeImageFormatTypeSize) { + case 8: + ChannelType = UR_IMAGE_CHANNEL_TYPE_SIGNED_INT8; + break; + case 16: + ChannelType = UR_IMAGE_CHANNEL_TYPE_SIGNED_INT16; + break; + case 32: + ChannelType = UR_IMAGE_CHANNEL_TYPE_SIGNED_INT32; + break; + default: + urPrint( + "ze2urImageFormat: unexpected image format type size: size = %zu\n", + ZeImageFormatTypeSize); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + case ZE_IMAGE_FORMAT_TYPE_UNORM: + switch (ZeImageFormatTypeSize) { + case 8: + ChannelType = UR_IMAGE_CHANNEL_TYPE_UNORM_INT8; + break; + case 16: + ChannelType = UR_IMAGE_CHANNEL_TYPE_UNORM_INT16; + break; + default: + urPrint( + "ze2urImageFormat: unexpected image format type size: size = %zu\n", + ZeImageFormatTypeSize); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + case ZE_IMAGE_FORMAT_TYPE_SNORM: + switch (ZeImageFormatTypeSize) { + case 8: + ChannelType = UR_IMAGE_CHANNEL_TYPE_SNORM_INT8; + break; + case 16: + ChannelType = UR_IMAGE_CHANNEL_TYPE_SNORM_INT16; + break; + default: + urPrint( + "ze2urImageFormat: unexpected image format type size: size = %zu\n", + ZeImageFormatTypeSize); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + case ZE_IMAGE_FORMAT_TYPE_FLOAT: + switch (ZeImageFormatTypeSize) { + case 16: + ChannelType = UR_IMAGE_CHANNEL_TYPE_HALF_FLOAT; + break; + case 32: + ChannelType = UR_IMAGE_CHANNEL_TYPE_FLOAT; + break; + default: + urPrint( + "ze2urImageFormat: unexpected image format type size: size = %zu\n", + ZeImageFormatTypeSize); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + default: + urPrint("ze2urImageFormat: unsupported image format type: type = %d\n", + ZeImageFormat.type); + return UR_RESULT_ERROR_INVALID_VALUE; + } + + UrImageFormat->channelOrder = ChannelOrder; + UrImageFormat->channelType = ChannelType; + return UR_RESULT_SUCCESS; +} + +/// Construct ZE image desc from UR image format and desc. +ur_result_t ur2zeImageDesc(const ur_image_format_t *ImageFormat, + const ur_image_desc_t *ImageDesc, + ZeStruct &ZeImageDesc) { + auto [ZeImageFormatType, ZeImageFormatTypeSize] = + getImageFormatTypeAndSize(ImageFormat); + // TODO: populate the layout mapping + ze_image_format_layout_t ZeImageFormatLayout; + switch (ImageFormat->channelOrder) { + case UR_IMAGE_CHANNEL_ORDER_A: + case UR_IMAGE_CHANNEL_ORDER_R: { + switch (ZeImageFormatTypeSize) { + case 8: + ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_8; + break; + case 16: + ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_16; + break; + case 32: + ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_32; + break; + default: + urPrint("urMemImageCreate: unexpected data type Size\n"); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + } + case UR_IMAGE_CHANNEL_ORDER_RG: + case UR_IMAGE_CHANNEL_ORDER_RA: + case UR_IMAGE_CHANNEL_ORDER_RX: { + switch (ZeImageFormatTypeSize) { + case 8: + ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_8_8; + break; + case 16: + ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_16_16; + break; + case 32: + ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_32_32; + break; + default: + urPrint("urMemImageCreate: unexpected data type Size\n"); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + } + case UR_IMAGE_CHANNEL_ORDER_RGB: + case UR_IMAGE_CHANNEL_ORDER_RGX: { + switch (ZeImageFormatTypeSize) { + default: + urPrint("urMemImageCreate: unexpected data type Size\n"); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + } + case UR_IMAGE_CHANNEL_ORDER_RGBX: + case UR_IMAGE_CHANNEL_ORDER_RGBA: + case UR_IMAGE_CHANNEL_ORDER_ARGB: + case UR_IMAGE_CHANNEL_ORDER_BGRA: { + switch (ZeImageFormatTypeSize) { + case 8: + ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_8_8_8_8; + break; + case 16: + ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_16_16_16_16; + break; + case 32: + ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_32_32_32_32; + break; + default: + urPrint("urMemImageCreate: unexpected data type Size\n"); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + } + default: + urPrint("format layout = %d\n", ImageFormat->channelOrder); + die("urMemImageCreate: unsupported image format layout\n"); + break; + } + + ze_image_format_t ZeFormatDesc = { + ZeImageFormatLayout, ZeImageFormatType, + // TODO: are swizzles deducted from image_format->image_channel_order? + ZE_IMAGE_FORMAT_SWIZZLE_R, ZE_IMAGE_FORMAT_SWIZZLE_G, + ZE_IMAGE_FORMAT_SWIZZLE_B, ZE_IMAGE_FORMAT_SWIZZLE_A}; + + ze_image_type_t ZeImageType; + switch (ImageDesc->type) { + case UR_MEM_TYPE_IMAGE1D: + ZeImageType = ZE_IMAGE_TYPE_1D; + break; + case UR_MEM_TYPE_IMAGE2D: + ZeImageType = ZE_IMAGE_TYPE_2D; + break; + case UR_MEM_TYPE_IMAGE3D: + ZeImageType = ZE_IMAGE_TYPE_3D; + break; + case UR_MEM_TYPE_IMAGE1D_ARRAY: + ZeImageType = ZE_IMAGE_TYPE_1DARRAY; + break; + case UR_MEM_TYPE_IMAGE2D_ARRAY: + ZeImageType = ZE_IMAGE_TYPE_2DARRAY; + break; + default: + urPrint("urMemImageCreate: unsupported image type\n"); + return UR_RESULT_ERROR_INVALID_VALUE; + } + + ZeImageDesc.stype = ZE_STRUCTURE_TYPE_IMAGE_DESC; + ZeImageDesc.pNext = ImageDesc->pNext; + ZeImageDesc.arraylevels = ZeImageDesc.flags = 0; + ZeImageDesc.type = ZeImageType; + ZeImageDesc.format = ZeFormatDesc; + ZeImageDesc.width = ur_cast(ImageDesc->width); + ZeImageDesc.height = std::max(ur_cast(ImageDesc->height), 1ul); + ZeImageDesc.depth = std::max(ur_cast(ImageDesc->depth), 1ul); + ZeImageDesc.arraylevels = ur_cast(ImageDesc->arraySize); + ZeImageDesc.miplevels = ImageDesc->numMipLevel; + + return UR_RESULT_SUCCESS; +} + +/// Return element size in bytes of a pixel. +uint32_t getPixelSizeBytes(const ur_image_format_t *Format) { + uint32_t NumChannels = 0; + switch (Format->channelOrder) { + case UR_IMAGE_CHANNEL_ORDER_A: + case UR_IMAGE_CHANNEL_ORDER_R: + case UR_IMAGE_CHANNEL_ORDER_INTENSITY: + case UR_IMAGE_CHANNEL_ORDER_LUMINANCE: + case UR_IMAGE_CHANNEL_ORDER_FORCE_UINT32: + NumChannels = 1; + break; + case UR_IMAGE_CHANNEL_ORDER_RG: + case UR_IMAGE_CHANNEL_ORDER_RA: + case UR_IMAGE_CHANNEL_ORDER_RX: + NumChannels = 2; + break; + case UR_IMAGE_CHANNEL_ORDER_RGB: + case UR_IMAGE_CHANNEL_ORDER_RGX: + NumChannels = 3; + break; + case UR_IMAGE_CHANNEL_ORDER_RGBA: + case UR_IMAGE_CHANNEL_ORDER_BGRA: + case UR_IMAGE_CHANNEL_ORDER_ARGB: + case UR_IMAGE_CHANNEL_ORDER_ABGR: + case UR_IMAGE_CHANNEL_ORDER_RGBX: + case UR_IMAGE_CHANNEL_ORDER_SRGBA: + NumChannels = 4; + break; + default: + ur::unreachable(); + } + uint32_t ChannelTypeSizeInBytes = 0; + switch (Format->channelType) { + case UR_IMAGE_CHANNEL_TYPE_SNORM_INT8: + case UR_IMAGE_CHANNEL_TYPE_UNORM_INT8: + case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT8: + case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT8: + ChannelTypeSizeInBytes = 1; + break; + case UR_IMAGE_CHANNEL_TYPE_SNORM_INT16: + case UR_IMAGE_CHANNEL_TYPE_UNORM_INT16: + case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT16: + case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT16: + case UR_IMAGE_CHANNEL_TYPE_UNORM_SHORT_565: + case UR_IMAGE_CHANNEL_TYPE_UNORM_SHORT_555: + ChannelTypeSizeInBytes = 2; + break; + case UR_IMAGE_CHANNEL_TYPE_HALF_FLOAT: + case UR_IMAGE_CHANNEL_TYPE_INT_101010: + case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT32: + case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT32: + case UR_IMAGE_CHANNEL_TYPE_FLOAT: + case UR_IMAGE_CHANNEL_TYPE_FORCE_UINT32: + ChannelTypeSizeInBytes = 4; + break; + default: + ur::unreachable(); + } + return NumChannels * ChannelTypeSizeInBytes; +} + +} // namespace + +ur_result_t getImageRegionHelper(ze_image_desc_t ZeImageDesc, + ur_rect_offset_t *Origin, + ur_rect_region_t *Region, + ze_image_region_t &ZeRegion) { + UR_ASSERT(Origin, UR_RESULT_ERROR_INVALID_VALUE); + UR_ASSERT(Region, UR_RESULT_ERROR_INVALID_VALUE); + + if (ZeImageDesc.type == ZE_IMAGE_TYPE_1D) { + Region->height = 1; + Region->depth = 1; + } else if (ZeImageDesc.type == ZE_IMAGE_TYPE_2D || + ZeImageDesc.type == ZE_IMAGE_TYPE_1DARRAY) { + Region->depth = 1; + } + +#ifndef NDEBUG + UR_ASSERT((ZeImageDesc.type == ZE_IMAGE_TYPE_1D && Origin->y == 0 && + Origin->z == 0) || + (ZeImageDesc.type == ZE_IMAGE_TYPE_1DARRAY && Origin->z == 0) || + (ZeImageDesc.type == ZE_IMAGE_TYPE_2D && Origin->z == 0) || + (ZeImageDesc.type == ZE_IMAGE_TYPE_3D), + UR_RESULT_ERROR_INVALID_VALUE); + + UR_ASSERT(Region->width && Region->height && Region->depth, + UR_RESULT_ERROR_INVALID_VALUE); + UR_ASSERT( + (ZeImageDesc.type == ZE_IMAGE_TYPE_1D && Region->height == 1 && + Region->depth == 1) || + (ZeImageDesc.type == ZE_IMAGE_TYPE_1DARRAY && Region->depth == 1) || + (ZeImageDesc.type == ZE_IMAGE_TYPE_2D && Region->depth == 1) || + (ZeImageDesc.type == ZE_IMAGE_TYPE_3D), + UR_RESULT_ERROR_INVALID_VALUE); +#endif // !NDEBUG + + uint32_t OriginX = ur_cast(Origin->x); + uint32_t OriginY = ur_cast(Origin->y); + uint32_t OriginZ = ur_cast(Origin->z); + + uint32_t Width = ur_cast(Region->width); + uint32_t Height = ur_cast(Region->height); + uint32_t Depth = ur_cast(Region->depth); + + ZeRegion = {OriginX, OriginY, OriginZ, Width, Height, Depth}; + + return UR_RESULT_SUCCESS; +} + +std::pair +getImageFormatTypeAndSize(const ur_image_format_t *ImageFormat) { + ze_image_format_type_t ZeImageFormatType; + size_t ZeImageFormatTypeSize; + switch (ImageFormat->channelType) { + case UR_IMAGE_CHANNEL_TYPE_FLOAT: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_FLOAT; + ZeImageFormatTypeSize = 32; + break; + } + case UR_IMAGE_CHANNEL_TYPE_HALF_FLOAT: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_FLOAT; + ZeImageFormatTypeSize = 16; + break; + } + case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT32: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UINT; + ZeImageFormatTypeSize = 32; + break; + } + case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT16: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UINT; + ZeImageFormatTypeSize = 16; + break; + } + case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT8: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UINT; + ZeImageFormatTypeSize = 8; + break; + } + case UR_IMAGE_CHANNEL_TYPE_UNORM_INT16: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UNORM; + ZeImageFormatTypeSize = 16; + break; + } + case UR_IMAGE_CHANNEL_TYPE_UNORM_INT8: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UNORM; + ZeImageFormatTypeSize = 8; + break; + } + case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT32: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SINT; + ZeImageFormatTypeSize = 32; + break; + } + case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT16: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SINT; + ZeImageFormatTypeSize = 16; + break; + } + case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT8: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SINT; + ZeImageFormatTypeSize = 8; + break; + } + case UR_IMAGE_CHANNEL_TYPE_SNORM_INT16: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SNORM; + ZeImageFormatTypeSize = 16; + break; + } + case UR_IMAGE_CHANNEL_TYPE_SNORM_INT8: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SNORM; + ZeImageFormatTypeSize = 8; + break; + } + default: + urPrint("urMemImageCreate: unsupported image data type: data type = %d\n", + ImageFormat->channelType); + ur::unreachable(); + } + return {ZeImageFormatType, ZeImageFormatTypeSize}; +} UR_APIEXPORT ur_result_t UR_APICALL urUSMPitchedAllocExp( ur_context_handle_t hContext, ur_device_handle_t hDevice, const ur_usm_desc_t *pUSMDesc, ur_usm_pool_handle_t pool, size_t widthInBytes, size_t height, size_t elementSizeBytes, void **ppMem, size_t *pResultPitch) { - std::ignore = hContext; - std::ignore = hDevice; - std::ignore = pUSMDesc; - std::ignore = pool; - std::ignore = widthInBytes; - std::ignore = height; - std::ignore = elementSizeBytes; - std::ignore = ppMem; - std::ignore = pResultPitch; - urPrint("[UR][L0] %s function not implemented!\n", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + std::shared_lock Lock(hContext->Mutex); + + static std::once_flag InitFlag; + std::call_once(InitFlag, [&]() { + ze_driver_handle_t DriverHandle = hContext->getPlatform()->ZeDriver; + auto Result = zeDriverGetExtensionFunctionAddress( + DriverHandle, "zeMemGetPitchFor2dImage", + (void **)&zeMemGetPitchFor2dImageFunctionPtr); + if (Result != ZE_RESULT_SUCCESS) + urPrint("zeDriverGetExtensionFunctionAddress zeMemGetPitchFor2dImage " + "failed, err = %d\n", + Result); + }); + if (!zeMemGetPitchFor2dImageFunctionPtr) + return UR_RESULT_ERROR_INVALID_OPERATION; + + size_t Width = widthInBytes / elementSizeBytes; + size_t RowPitch; + ZE2UR_CALL(zeMemGetPitchFor2dImageFunctionPtr, + (hContext->ZeContext, hDevice->ZeDevice, Width, height, + elementSizeBytes, &RowPitch)); + *pResultPitch = RowPitch; + + size_t Size = height * RowPitch; + UR_CALL(urUSMDeviceAlloc(hContext, hDevice, pUSMDesc, pool, Size, ppMem)); + + return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL @@ -36,32 +595,50 @@ urBindlessImagesUnsampledImageHandleDestroyExp(ur_context_handle_t hContext, std::ignore = hContext; std::ignore = hDevice; std::ignore = hImage; - urPrint("[UR][L0] %s function not implemented!\n", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + + return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesSampledImageHandleDestroyExp(ur_context_handle_t hContext, ur_device_handle_t hDevice, ur_exp_image_handle_t hImage) { - std::ignore = hContext; - std::ignore = hDevice; - std::ignore = hImage; - urPrint("[UR][L0] %s function not implemented!\n", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + // Sampled image is a combination of unsampled image and sampler. + urBindlessImagesUnsampledImageHandleDestroyExp(hContext, hDevice, hImage); + + return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesImageAllocateExp( ur_context_handle_t hContext, ur_device_handle_t hDevice, const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, ur_exp_image_mem_handle_t *phImageMem) { - std::ignore = hContext; - std::ignore = hDevice; - std::ignore = pImageFormat; - std::ignore = pImageDesc; - std::ignore = phImageMem; - urPrint("[UR][L0] %s function not implemented!\n", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + std::shared_lock Lock(hContext->Mutex); + + ZeStruct ZeImageDesc; + UR_CALL(ur2zeImageDesc(pImageFormat, pImageDesc, ZeImageDesc)); + + ze_image_bindless_exp_desc_t ZeImageBindlessDesc; + ZeImageBindlessDesc.stype = ZE_STRUCTURE_TYPE_BINDLESS_IMAGE_EXP_DESC; + ZeImageBindlessDesc.pNext = nullptr; + ZeImageBindlessDesc.flags = ZE_IMAGE_BINDLESS_EXP_FLAG_BINDLESS; + ZeImageDesc.pNext = &ZeImageBindlessDesc; + + ze_image_handle_t ZeImage; + ZE2UR_CALL(zeImageCreate, + (hContext->ZeContext, hDevice->ZeDevice, &ZeImageDesc, &ZeImage)); + ZE2UR_CALL(zeContextMakeImageResident, + (hContext->ZeContext, hDevice->ZeDevice, ZeImage)); + try { + auto UrImage = new _ur_image(hContext, ZeImage, /*OwnZeMemHandle*/ true); + UrImage->ZeImageDesc = ZeImageDesc; + *phImageMem = reinterpret_cast(UrImage); + } catch (const std::bad_alloc &) { + return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } catch (...) { + return UR_RESULT_ERROR_UNKNOWN; + } + return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesImageFreeExp( @@ -69,9 +646,18 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesImageFreeExp( ur_exp_image_mem_handle_t hImageMem) { std::ignore = hContext; std::ignore = hDevice; - std::ignore = hImageMem; - urPrint("[UR][L0] %s function not implemented!\n", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + auto *UrImage = reinterpret_cast<_ur_image *>(hImageMem); + if (!UrImage->RefCount.decrementAndTest()) + return UR_RESULT_SUCCESS; + + if (UrImage->OwnNativeHandle) { + auto ZeResult = ZE_CALL_NOCHECK(zeImageDestroy, (UrImage->ZeImage)); + // Gracefully handle the case that L0 was already unloaded. + if (ZeResult && ZeResult != ZE_RESULT_ERROR_UNINITIALIZED) + return ze2urResult(ZeResult); + } + delete UrImage; + return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesUnsampledImageCreateExp( @@ -79,15 +665,87 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesUnsampledImageCreateExp( ur_exp_image_mem_handle_t hImageMem, const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, ur_mem_handle_t *phMem, ur_exp_image_handle_t *phImage) { - std::ignore = hContext; - std::ignore = hDevice; - std::ignore = hImageMem; - std::ignore = pImageFormat; - std::ignore = pImageDesc; std::ignore = phMem; - std::ignore = phImage; - urPrint("[UR][L0] %s function not implemented!\n", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + std::shared_lock Lock(hContext->Mutex); + + ZeStruct ZeImageDesc; + UR_CALL(ur2zeImageDesc(pImageFormat, pImageDesc, ZeImageDesc)); + + _ur_image *UrImage = nullptr; + + ze_memory_allocation_properties_t MemAllocProperties{ + ZE_STRUCTURE_TYPE_MEMORY_ALLOCATION_PROPERTIES}; + ZE2UR_CALL(zeMemGetAllocProperties, + (hContext->ZeContext, hImageMem, &MemAllocProperties, nullptr)); + if (MemAllocProperties.type == ZE_MEMORY_TYPE_UNKNOWN) { + UrImage = reinterpret_cast<_ur_image *>(hImageMem); + if (!isSameImageDesc(&UrImage->ZeImageDesc, &ZeImageDesc)) { + ze_image_bindless_exp_desc_t ZeImageBindlessDesc; + ZeImageBindlessDesc.stype = ZE_STRUCTURE_TYPE_BINDLESS_IMAGE_EXP_DESC; + ZeImageBindlessDesc.pNext = nullptr; + ZeImageBindlessDesc.flags = ZE_IMAGE_BINDLESS_EXP_FLAG_BINDLESS; + ZeImageDesc.pNext = &ZeImageBindlessDesc; + ze_image_handle_t ZeImageView; + ZE2UR_CALL(zeImageViewCreateExt, + (hContext->ZeContext, hDevice->ZeDevice, &ZeImageDesc, + UrImage->ZeImage, &ZeImageView)); + ZE2UR_CALL(zeContextMakeImageResident, + (hContext->ZeContext, hDevice->ZeDevice, ZeImageView)); + try { + UrImage = new _ur_image(hContext, ZeImageView, /*OwnZeMemHandle*/ true); + UrImage->ZeImageDesc = ZeImageDesc; + *phMem = UrImage; + } catch (const std::bad_alloc &) { + return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } catch (...) { + return UR_RESULT_ERROR_UNKNOWN; + } + } else { + *phMem = nullptr; + } + } else { + ze_image_pitched_exp_desc_t ZeImagePitchedDesc; + ZeImagePitchedDesc.stype = ZE_STRUCTURE_TYPE_PITCHED_IMAGE_EXP_DESC; + ZeImagePitchedDesc.pNext = nullptr; + ZeImagePitchedDesc.ptr = hImageMem; + ZeImageDesc.pNext = &ZeImagePitchedDesc; + + ze_image_handle_t ZeImage; + ZE2UR_CALL(zeImageCreate, (hContext->ZeContext, hDevice->ZeDevice, + &ZeImageDesc, &ZeImage)); + ZE2UR_CALL(zeContextMakeImageResident, + (hContext->ZeContext, hDevice->ZeDevice, ZeImage)); + try { + UrImage = new _ur_image(hContext, ZeImage, /*OwnZeMemHandle*/ true); + UrImage->ZeImageDesc = ZeImageDesc; + *phMem = UrImage; + } catch (const std::bad_alloc &) { + return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } catch (...) { + return UR_RESULT_ERROR_UNKNOWN; + } + } + + static std::once_flag InitFlag; + std::call_once(InitFlag, [&]() { + ze_driver_handle_t DriverHandle = hContext->getPlatform()->ZeDriver; + auto Result = zeDriverGetExtensionFunctionAddress( + DriverHandle, "zeImageGetDeviceOffsetExp", + (void **)&zeImageGetDeviceOffsetExpFunctionPtr); + if (Result != ZE_RESULT_SUCCESS) + urPrint("zeDriverGetExtensionFunctionAddress zeImageGetDeviceOffsetExp " + "failed, err = %d\n", + Result); + }); + if (!zeImageGetDeviceOffsetExpFunctionPtr) + return UR_RESULT_ERROR_INVALID_OPERATION; + + uint64_t DeviceOffset; + ZE2UR_CALL(zeImageGetDeviceOffsetExpFunctionPtr, + (UrImage->ZeImage, &DeviceOffset)); + *phImage = reinterpret_cast(DeviceOffset); + + return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesSampledImageCreateExp( @@ -95,16 +753,20 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesSampledImageCreateExp( ur_exp_image_mem_handle_t hImageMem, const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, ur_sampler_handle_t hSampler, ur_mem_handle_t *phMem, ur_exp_image_handle_t *phImage) { - std::ignore = hContext; - std::ignore = hDevice; - std::ignore = hImageMem; - std::ignore = pImageFormat; - std::ignore = pImageDesc; - std::ignore = hSampler; - std::ignore = phMem; - std::ignore = phImage; - urPrint("[UR][L0] %s function not implemented!\n", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + + UR_CALL(urBindlessImagesUnsampledImageCreateExp( + hContext, hDevice, hImageMem, pImageFormat, pImageDesc, phMem, phImage)); + struct combined_sampled_image_handle { + uint64_t raw_image_handle; + uint64_t raw_sampler_handle; + }; + combined_sampled_image_handle *sampledImageHandle = + reinterpret_cast(phImage); + sampledImageHandle->raw_image_handle = reinterpret_cast(*phImage); + sampledImageHandle->raw_sampler_handle = + reinterpret_cast(hSampler->ZeSampler); + + return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesImageCopyExp( @@ -114,32 +776,149 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesImageCopyExp( ur_rect_offset_t dstOffset, ur_rect_region_t copyExtent, ur_rect_region_t hostExtent, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { - std::ignore = hQueue; - std::ignore = pDst; - std::ignore = pSrc; - std::ignore = pImageFormat; - std::ignore = pImageDesc; - std::ignore = imageCopyFlags; - std::ignore = srcOffset; - std::ignore = dstOffset; - std::ignore = copyExtent; - std::ignore = hostExtent; - std::ignore = numEventsInWaitList; - std::ignore = phEventWaitList; - std::ignore = phEvent; - urPrint("[UR][L0] %s function not implemented!\n", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + std::scoped_lock Lock(hQueue->Mutex); + ZeStruct ZeImageDesc; + UR_CALL(ur2zeImageDesc(pImageFormat, pImageDesc, ZeImageDesc)); + + bool UseCopyEngine = hQueue->useCopyEngine(/*PreferCopyEngine*/ true); + + _ur_ze_event_list_t TmpWaitList; + UR_CALL(TmpWaitList.createAndRetainUrZeEventList( + numEventsInWaitList, phEventWaitList, hQueue, UseCopyEngine)); + + bool Blocking = false; + // We want to batch these commands to avoid extra submissions (costly) + bool OkToBatch = true; + + // Get a new command list to be used on this call + ur_command_list_ptr_t CommandList{}; + UR_CALL(hQueue->Context->getAvailableCommandList(hQueue, CommandList, + UseCopyEngine, OkToBatch)); + + ze_event_handle_t ZeEvent = nullptr; + ur_event_handle_t InternalEvent; + bool IsInternal = phEvent == nullptr; + ur_event_handle_t *Event = phEvent ? phEvent : &InternalEvent; + UR_CALL(createEventAndAssociateQueue(hQueue, Event, UR_COMMAND_MEM_IMAGE_COPY, + CommandList, IsInternal, + /*IsMultiDevice*/ false)); + ZeEvent = (*Event)->ZeEvent; + (*Event)->WaitList = TmpWaitList; + + const auto &ZeCommandList = CommandList->first; + const auto &WaitList = (*Event)->WaitList; + + if (imageCopyFlags == UR_EXP_IMAGE_COPY_FLAG_HOST_TO_DEVICE) { + if (pImageDesc->rowPitch == 0) { + // Copy to Non-USM memory + ze_image_region_t DstRegion; + UR_CALL(getImageRegionHelper(ZeImageDesc, &dstOffset, ©Extent, + DstRegion)); + auto *UrImage = static_cast<_ur_image *>(pDst); + ZE2UR_CALL(zeCommandListAppendImageCopyFromMemory, + (ZeCommandList, UrImage->ZeImage, pSrc, &DstRegion, ZeEvent, + WaitList.Length, WaitList.ZeEventList)); + } else { + // Copy to pitched USM memory + uint32_t DstPitch = pImageDesc->rowPitch; + ze_copy_region_t ZeDstRegion = { + (uint32_t)dstOffset.x, (uint32_t)dstOffset.y, + (uint32_t)dstOffset.z, DstPitch, + (uint32_t)copyExtent.height, (uint32_t)copyExtent.depth}; + uint32_t DstSlicePitch = 0; + uint32_t SrcPitch = hostExtent.width * getPixelSizeBytes(pImageFormat); + ze_copy_region_t ZeSrcRegion = { + (uint32_t)srcOffset.x, (uint32_t)srcOffset.y, + (uint32_t)srcOffset.z, SrcPitch, + (uint32_t)copyExtent.height, (uint32_t)copyExtent.depth}; + uint32_t SrcSlicePitch = 0; + ZE2UR_CALL(zeCommandListAppendMemoryCopyRegion, + (ZeCommandList, pDst, &ZeDstRegion, DstPitch, DstSlicePitch, + pSrc, &ZeSrcRegion, SrcPitch, SrcSlicePitch, ZeEvent, + WaitList.Length, WaitList.ZeEventList)); + } + } else if (imageCopyFlags == UR_EXP_IMAGE_COPY_FLAG_DEVICE_TO_HOST) { + if (pImageDesc->rowPitch == 0) { + // Copy from Non-USM memory to host + ze_image_region_t SrcRegion; + UR_CALL(getImageRegionHelper(ZeImageDesc, &srcOffset, ©Extent, + SrcRegion)); + auto *UrImage = static_cast<_ur_image *>(pSrc); + ZE2UR_CALL(zeCommandListAppendImageCopyToMemory, + (ZeCommandList, pDst, UrImage->ZeImage, &SrcRegion, ZeEvent, + WaitList.Length, WaitList.ZeEventList)); + } else { + // Copy from pitched USM memory to host + uint32_t DstPitch = copyExtent.width * getPixelSizeBytes(pImageFormat); + ze_copy_region_t ZeDstRegion = { + (uint32_t)dstOffset.x, (uint32_t)dstOffset.y, + (uint32_t)dstOffset.z, DstPitch, + (uint32_t)copyExtent.height, (uint32_t)copyExtent.depth}; + uint32_t DstSlicePitch = 0; + uint32_t SrcPitch = pImageDesc->rowPitch; + ze_copy_region_t ZeSrcRegion = { + (uint32_t)srcOffset.x, (uint32_t)srcOffset.y, + (uint32_t)srcOffset.z, SrcPitch, + (uint32_t)copyExtent.height, (uint32_t)copyExtent.depth}; + uint32_t SrcSlicePitch = 0; + ZE2UR_CALL(zeCommandListAppendMemoryCopyRegion, + (ZeCommandList, pDst, &ZeDstRegion, DstPitch, DstSlicePitch, + pSrc, &ZeSrcRegion, SrcPitch, SrcSlicePitch, ZeEvent, + WaitList.Length, WaitList.ZeEventList)); + } + } else { + urPrint("urBindlessImagesImageCopyExp: unexpected imageCopyFlags\n"); + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + + UR_CALL(hQueue->executeCommandList(CommandList, Blocking, OkToBatch)); + + return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesImageGetInfoExp( ur_exp_image_mem_handle_t hImageMem, ur_image_info_t propName, void *pPropValue, size_t *pPropSizeRet) { - std::ignore = hImageMem; - std::ignore = propName; - std::ignore = pPropValue; - std::ignore = pPropSizeRet; - urPrint("[UR][L0] %s function not implemented!\n", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + auto *UrImage = reinterpret_cast<_ur_image *>(hImageMem); + ze_image_desc_t &Desc = UrImage->ZeImageDesc; + switch (propName) { + case UR_IMAGE_INFO_WIDTH: + if (pPropValue) { + *(uint64_t *)pPropValue = Desc.width; + } + if (pPropSizeRet) { + *pPropSizeRet = sizeof(uint64_t); + } + return UR_RESULT_SUCCESS; + case UR_IMAGE_INFO_HEIGHT: + if (pPropValue) { + *(uint32_t *)pPropValue = Desc.height; + } + if (pPropSizeRet) { + *pPropSizeRet = sizeof(uint32_t); + } + return UR_RESULT_SUCCESS; + case UR_IMAGE_INFO_DEPTH: + if (pPropValue) { + *(uint32_t *)pPropValue = Desc.depth; + } + if (pPropSizeRet) { + *pPropSizeRet = sizeof(uint32_t); + } + return UR_RESULT_SUCCESS; + case UR_IMAGE_INFO_FORMAT: + if (pPropValue) { + ur_image_format_t UrImageFormat; + UR_CALL(ze2urImageFormat(&Desc, &UrImageFormat)); + *(ur_image_format_t *)pPropValue = UrImageFormat; + } + if (pPropSizeRet) { + *pPropSizeRet = sizeof(ur_image_format_t); + } + return UR_RESULT_SUCCESS; + default: + return UR_RESULT_ERROR_INVALID_VALUE; + } } UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesMipmapGetLevelExp( diff --git a/source/adapters/level_zero/image.hpp b/source/adapters/level_zero/image.hpp index d579a24708..618258601d 100644 --- a/source/adapters/level_zero/image.hpp +++ b/source/adapters/level_zero/image.hpp @@ -13,3 +13,11 @@ #include #include #include + +ur_result_t getImageRegionHelper(ze_image_desc_t ZeImageDesc, + ur_rect_offset_t *Origin, + ur_rect_region_t *Region, + ze_image_region_t &ZeRegion); + +std::pair +getImageFormatTypeAndSize(const ur_image_format_t *ImageFormat); diff --git a/source/adapters/level_zero/memory.cpp b/source/adapters/level_zero/memory.cpp index e977d1ac15..dc7d60bd2e 100644 --- a/source/adapters/level_zero/memory.cpp +++ b/source/adapters/level_zero/memory.cpp @@ -14,6 +14,7 @@ #include "context.hpp" #include "event.hpp" +#include "image.hpp" #include "ur_level_zero.hpp" // Default to using compute engine for fill operation, but allow to @@ -287,49 +288,6 @@ static ur_result_t ZeHostMemAllocHelper(void **ResultPtr, return UR_RESULT_SUCCESS; } -static ur_result_t getImageRegionHelper(_ur_image *Mem, - ur_rect_offset_t *Origin, - ur_rect_region_t *Region, - ze_image_region_t &ZeRegion) { - UR_ASSERT(Mem, UR_RESULT_ERROR_INVALID_MEM_OBJECT); - UR_ASSERT(Origin, UR_RESULT_ERROR_INVALID_VALUE); - -#ifndef NDEBUG - auto UrImage = static_cast<_ur_image *>(Mem); - ze_image_desc_t &ZeImageDesc = UrImage->ZeImageDesc; - - UR_ASSERT(Mem->isImage(), UR_RESULT_ERROR_INVALID_MEM_OBJECT); - UR_ASSERT((ZeImageDesc.type == ZE_IMAGE_TYPE_1D && Origin->y == 0 && - Origin->z == 0) || - (ZeImageDesc.type == ZE_IMAGE_TYPE_1DARRAY && Origin->z == 0) || - (ZeImageDesc.type == ZE_IMAGE_TYPE_2D && Origin->z == 0) || - (ZeImageDesc.type == ZE_IMAGE_TYPE_3D), - UR_RESULT_ERROR_INVALID_VALUE); - - UR_ASSERT(Region->width && Region->height && Region->depth, - UR_RESULT_ERROR_INVALID_VALUE); - UR_ASSERT( - (ZeImageDesc.type == ZE_IMAGE_TYPE_1D && Region->height == 1 && - Region->depth == 1) || - (ZeImageDesc.type == ZE_IMAGE_TYPE_1DARRAY && Region->depth == 1) || - (ZeImageDesc.type == ZE_IMAGE_TYPE_2D && Region->depth == 1) || - (ZeImageDesc.type == ZE_IMAGE_TYPE_3D), - UR_RESULT_ERROR_INVALID_VALUE); -#endif // !NDEBUG - - uint32_t OriginX = ur_cast(Origin->x); - uint32_t OriginY = ur_cast(Origin->y); - uint32_t OriginZ = ur_cast(Origin->z); - - uint32_t Width = ur_cast(Region->width); - uint32_t Height = ur_cast(Region->height); - uint32_t Depth = ur_cast(Region->depth); - - ZeRegion = {OriginX, OriginY, OriginZ, Width, Height, Depth}; - - return UR_RESULT_SUCCESS; -} - // Helper function to implement image read/write/copy. // PI interfaces must have queue's and destination image's mutexes locked for // exclusive use and source image's mutex locked for shared use on entry. @@ -372,7 +330,8 @@ static ur_result_t enqueueMemImageCommandHelper( _ur_image *SrcMem = ur_cast<_ur_image *>(const_cast(Src)); ze_image_region_t ZeSrcRegion; - UR_CALL(getImageRegionHelper(SrcMem, SrcOrigin, Region, ZeSrcRegion)); + UR_CALL(getImageRegionHelper(SrcMem->ZeImageDesc, SrcOrigin, Region, + ZeSrcRegion)); // TODO: Level Zero does not support row_pitch/slice_pitch for images yet. // Check that SYCL RT did not want pitch larger than default. @@ -406,7 +365,8 @@ static ur_result_t enqueueMemImageCommandHelper( } else if (CommandType == UR_COMMAND_MEM_IMAGE_WRITE) { _ur_image *DstMem = ur_cast<_ur_image *>(Dst); ze_image_region_t ZeDstRegion; - UR_CALL(getImageRegionHelper(DstMem, DstOrigin, Region, ZeDstRegion)); + UR_CALL(getImageRegionHelper(DstMem->ZeImageDesc, DstOrigin, Region, + ZeDstRegion)); // TODO: Level Zero does not support row_pitch/slice_pitch for images yet. // Check that SYCL RT did not want pitch larger than default. @@ -440,9 +400,11 @@ static ur_result_t enqueueMemImageCommandHelper( _ur_image *DstImage = ur_cast<_ur_image *>(Dst); ze_image_region_t ZeSrcRegion; - UR_CALL(getImageRegionHelper(SrcImage, SrcOrigin, Region, ZeSrcRegion)); + UR_CALL(getImageRegionHelper(SrcImage->ZeImageDesc, SrcOrigin, Region, + ZeSrcRegion)); ze_image_region_t ZeDstRegion; - UR_CALL(getImageRegionHelper(DstImage, DstOrigin, Region, ZeDstRegion)); + UR_CALL(getImageRegionHelper(DstImage->ZeImageDesc, DstOrigin, Region, + ZeDstRegion)); char *ZeHandleSrc = nullptr; char *ZeHandleDst = nullptr; @@ -1456,74 +1418,8 @@ static ur_result_t ur2zeImageDesc(const ur_image_format_t *ImageFormat, const ur_image_desc_t *ImageDesc, ZeStruct &ZeImageDesc) { - ze_image_format_type_t ZeImageFormatType; - size_t ZeImageFormatTypeSize; - switch (ImageFormat->channelType) { - case UR_IMAGE_CHANNEL_TYPE_FLOAT: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_FLOAT; - ZeImageFormatTypeSize = 32; - break; - } - case UR_IMAGE_CHANNEL_TYPE_HALF_FLOAT: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_FLOAT; - ZeImageFormatTypeSize = 16; - break; - } - case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT32: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UINT; - ZeImageFormatTypeSize = 32; - break; - } - case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT16: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UINT; - ZeImageFormatTypeSize = 16; - break; - } - case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT8: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UINT; - ZeImageFormatTypeSize = 8; - break; - } - case UR_IMAGE_CHANNEL_TYPE_UNORM_INT16: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UNORM; - ZeImageFormatTypeSize = 16; - break; - } - case UR_IMAGE_CHANNEL_TYPE_UNORM_INT8: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UNORM; - ZeImageFormatTypeSize = 8; - break; - } - case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT32: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SINT; - ZeImageFormatTypeSize = 32; - break; - } - case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT16: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SINT; - ZeImageFormatTypeSize = 16; - break; - } - case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT8: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SINT; - ZeImageFormatTypeSize = 8; - break; - } - case UR_IMAGE_CHANNEL_TYPE_SNORM_INT16: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SNORM; - ZeImageFormatTypeSize = 16; - break; - } - case UR_IMAGE_CHANNEL_TYPE_SNORM_INT8: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SNORM; - ZeImageFormatTypeSize = 8; - break; - } - default: - urPrint("urMemImageCreate: unsupported image data type: data type = %d\n", - ImageFormat->channelType); - return UR_RESULT_ERROR_INVALID_VALUE; - } + auto [ZeImageFormatType, ZeImageFormatTypeSize] = + getImageFormatTypeAndSize(ImageFormat); // TODO: populate the layout mapping ze_image_format_layout_t ZeImageFormatLayout; diff --git a/source/adapters/level_zero/memory.hpp b/source/adapters/level_zero/memory.hpp index 8efd5b136e..cd8bb42f41 100644 --- a/source/adapters/level_zero/memory.hpp +++ b/source/adapters/level_zero/memory.hpp @@ -206,10 +206,8 @@ struct _ur_image final : ur_mem_handle_t_ { bool isImage() const override { return true; } -#ifndef NDEBUG - // Keep the descriptor of the image (for debugging purposes) + // Keep the descriptor of the image ZeStruct ZeImageDesc; -#endif // !NDEBUG // Level Zero image handle. ze_image_handle_t ZeImage; From 400f08afe22244168fb2eca3507f936626434fff Mon Sep 17 00:00:00 2001 From: "He, Wenju" Date: Tue, 27 Feb 2024 17:14:01 -0800 Subject: [PATCH 03/11] 1D_USM not supported, fix build --- source/adapters/level_zero/device.cpp | 2 +- source/adapters/level_zero/image.cpp | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/source/adapters/level_zero/device.cpp b/source/adapters/level_zero/device.cpp index 88916329ad..2f6b3a91ff 100644 --- a/source/adapters/level_zero/device.cpp +++ b/source/adapters/level_zero/device.cpp @@ -924,7 +924,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo( case UR_DEVICE_INFO_BINDLESS_IMAGES_SHARED_USM_SUPPORT_EXP: return ReturnValue(true); case UR_DEVICE_INFO_BINDLESS_IMAGES_1D_USM_SUPPORT_EXP: - return ReturnValue(true); + return ReturnValue(false); case UR_DEVICE_INFO_BINDLESS_IMAGES_2D_USM_SUPPORT_EXP: return ReturnValue(true); case UR_DEVICE_INFO_IMAGE_PITCH_ALIGN_EXP: diff --git a/source/adapters/level_zero/image.cpp b/source/adapters/level_zero/image.cpp index 6ff38672b7..ac2502a2fb 100644 --- a/source/adapters/level_zero/image.cpp +++ b/source/adapters/level_zero/image.cpp @@ -298,6 +298,15 @@ ur_result_t ur2zeImageDesc(const ur_image_format_t *ImageFormat, case UR_IMAGE_CHANNEL_ORDER_RGX: { switch (ZeImageFormatTypeSize) { default: + case 8: + ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_8_8_8; + break; + case 16: + ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_16_16_16; + break; + case 32: + ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_32_32_32; + break; urPrint("urMemImageCreate: unexpected data type Size\n"); return UR_RESULT_ERROR_INVALID_VALUE; } @@ -363,8 +372,8 @@ ur_result_t ur2zeImageDesc(const ur_image_format_t *ImageFormat, ZeImageDesc.type = ZeImageType; ZeImageDesc.format = ZeFormatDesc; ZeImageDesc.width = ur_cast(ImageDesc->width); - ZeImageDesc.height = std::max(ur_cast(ImageDesc->height), 1ul); - ZeImageDesc.depth = std::max(ur_cast(ImageDesc->depth), 1ul); + ZeImageDesc.height = std::max(ur_cast(ImageDesc->height), (uint64_t)1); + ZeImageDesc.depth = std::max(ur_cast(ImageDesc->depth), (uint64_t)1); ZeImageDesc.arraylevels = ur_cast(ImageDesc->arraySize); ZeImageDesc.miplevels = ImageDesc->numMipLevel; From abfa49c04f94f9c3dc12f6597526951c120bdc41 Mon Sep 17 00:00:00 2001 From: "He, Wenju" Date: Tue, 27 Feb 2024 17:42:30 -0800 Subject: [PATCH 04/11] UR_IMAGE_CHANNEL_ORDER_RGB UR_IMAGE_CHANNEL_ORDER_RGX not supported --- source/adapters/level_zero/image.cpp | 30 ++++++---------------------- 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/source/adapters/level_zero/image.cpp b/source/adapters/level_zero/image.cpp index ac2502a2fb..078c718cc3 100644 --- a/source/adapters/level_zero/image.cpp +++ b/source/adapters/level_zero/image.cpp @@ -270,7 +270,7 @@ ur_result_t ur2zeImageDesc(const ur_image_format_t *ImageFormat, ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_32; break; default: - urPrint("urMemImageCreate: unexpected data type Size\n"); + urPrint("ur2zeImageDesc: unexpected data type size\n"); return UR_RESULT_ERROR_INVALID_VALUE; } break; @@ -289,25 +289,7 @@ ur_result_t ur2zeImageDesc(const ur_image_format_t *ImageFormat, ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_32_32; break; default: - urPrint("urMemImageCreate: unexpected data type Size\n"); - return UR_RESULT_ERROR_INVALID_VALUE; - } - break; - } - case UR_IMAGE_CHANNEL_ORDER_RGB: - case UR_IMAGE_CHANNEL_ORDER_RGX: { - switch (ZeImageFormatTypeSize) { - default: - case 8: - ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_8_8_8; - break; - case 16: - ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_16_16_16; - break; - case 32: - ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_32_32_32; - break; - urPrint("urMemImageCreate: unexpected data type Size\n"); + urPrint("ur2zeImageDesc: unexpected data type size\n"); return UR_RESULT_ERROR_INVALID_VALUE; } break; @@ -327,14 +309,14 @@ ur_result_t ur2zeImageDesc(const ur_image_format_t *ImageFormat, ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_32_32_32_32; break; default: - urPrint("urMemImageCreate: unexpected data type Size\n"); + urPrint("ur2zeImageDesc: unexpected data type size\n"); return UR_RESULT_ERROR_INVALID_VALUE; } break; } default: - urPrint("format layout = %d\n", ImageFormat->channelOrder); - die("urMemImageCreate: unsupported image format layout\n"); + urPrint("format channel order = %d\n", ImageFormat->channelOrder); + die("ur2zeImageDesc: unsupported image channel order\n"); break; } @@ -362,7 +344,7 @@ ur_result_t ur2zeImageDesc(const ur_image_format_t *ImageFormat, ZeImageType = ZE_IMAGE_TYPE_2DARRAY; break; default: - urPrint("urMemImageCreate: unsupported image type\n"); + urPrint("ur2zeImageDesc: unsupported image type\n"); return UR_RESULT_ERROR_INVALID_VALUE; } From 8b9e9945c00cf0d8a1ca9b40f54fcd715241b415 Mon Sep 17 00:00:00 2001 From: "He, Wenju" Date: Wed, 28 Feb 2024 02:51:56 -0800 Subject: [PATCH 05/11] PitchedDesc, UR_CALL --- source/adapters/level_zero/image.cpp | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/source/adapters/level_zero/image.cpp b/source/adapters/level_zero/image.cpp index 078c718cc3..251ddf78b6 100644 --- a/source/adapters/level_zero/image.cpp +++ b/source/adapters/level_zero/image.cpp @@ -595,7 +595,8 @@ urBindlessImagesSampledImageHandleDestroyExp(ur_context_handle_t hContext, ur_device_handle_t hDevice, ur_exp_image_handle_t hImage) { // Sampled image is a combination of unsampled image and sampler. - urBindlessImagesUnsampledImageHandleDestroyExp(hContext, hDevice, hImage); + UR_CALL(urBindlessImagesUnsampledImageHandleDestroyExp(hContext, hDevice, + hImage)); return UR_RESULT_SUCCESS; } @@ -695,11 +696,17 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesUnsampledImageCreateExp( *phMem = nullptr; } } else { - ze_image_pitched_exp_desc_t ZeImagePitchedDesc; - ZeImagePitchedDesc.stype = ZE_STRUCTURE_TYPE_PITCHED_IMAGE_EXP_DESC; - ZeImagePitchedDesc.pNext = nullptr; - ZeImagePitchedDesc.ptr = hImageMem; - ZeImageDesc.pNext = &ZeImagePitchedDesc; + ze_image_pitched_exp_desc_t PitchedDesc; + PitchedDesc.stype = ZE_STRUCTURE_TYPE_PITCHED_IMAGE_EXP_DESC; + PitchedDesc.pNext = nullptr; + PitchedDesc.ptr = hImageMem; + + ze_image_bindless_exp_desc_t BindlessDesc; + BindlessDesc.stype = ZE_STRUCTURE_TYPE_BINDLESS_IMAGE_EXP_DESC; + BindlessDesc.pNext = &PitchedDesc; + BindlessDesc.flags = ZE_IMAGE_BINDLESS_EXP_FLAG_BINDLESS; + + ZeImageDesc.pNext = &BindlessDesc; ze_image_handle_t ZeImage; ZE2UR_CALL(zeImageCreate, (hContext->ZeContext, hDevice->ZeDevice, @@ -747,6 +754,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesSampledImageCreateExp( UR_CALL(urBindlessImagesUnsampledImageCreateExp( hContext, hDevice, hImageMem, pImageFormat, pImageDesc, phMem, phImage)); + struct combined_sampled_image_handle { uint64_t raw_image_handle; uint64_t raw_sampler_handle; From 753f15af64b1c812c42a6bd8bf0a30653a02b270 Mon Sep 17 00:00:00 2001 From: "Kenneth Benzie (Benie)" Date: Wed, 28 Feb 2024 15:02:14 +0000 Subject: [PATCH 06/11] [SPEC] Clarify the UR_PROGRAM_INFO_DEVICES query Fixes #1384 by clarifying what the *list of devices associated with a program* means when the program is created in various ways. --- include/ur_api.h | 5 +++-- scripts/core/program.yml | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/include/ur_api.h b/include/ur_api.h index 7ba79f4e13..6291dbcd6d 100644 --- a/include/ur_api.h +++ b/include/ur_api.h @@ -4336,8 +4336,9 @@ typedef enum ur_program_info_t { ///< provided for identifying memory leaks. UR_PROGRAM_INFO_CONTEXT = 1, ///< [::ur_context_handle_t] Program context info. UR_PROGRAM_INFO_NUM_DEVICES = 2, ///< [uint32_t] Return number of devices associated with Program. - UR_PROGRAM_INFO_DEVICES = 3, ///< [::ur_device_handle_t[]] Return list of devices associated with - ///< Program. + UR_PROGRAM_INFO_DEVICES = 3, ///< [::ur_device_handle_t[]] Return list of devices associated with a program. + ///< This is either the list of devices associated with the context or a + ///< subset of those devices when the program is created using ::urProgramCreateWithBinary. UR_PROGRAM_INFO_SOURCE = 4, ///< [char[]] Return program source associated with Program. UR_PROGRAM_INFO_BINARY_SIZES = 5, ///< [size_t[]] Return program binary sizes for each device. UR_PROGRAM_INFO_BINARIES = 6, ///< [unsigned char[]] Return program binaries for all devices for this diff --git a/scripts/core/program.yml b/scripts/core/program.yml index 88b652210b..f642423af9 100644 --- a/scripts/core/program.yml +++ b/scripts/core/program.yml @@ -327,7 +327,9 @@ etors: - name: NUM_DEVICES desc: "[uint32_t] Return number of devices associated with Program." - name: DEVICES - desc: "[$x_device_handle_t[]] Return list of devices associated with Program." + desc: | + [$x_device_handle_t[]] Return list of devices associated with a program. + This is either the list of devices associated with the context or a subset of those devices when the program is created using $xProgramCreateWithBinary. - name: SOURCE desc: "[char[]] Return program source associated with Program." - name: BINARY_SIZES From c515c5594e86e231751f29c3bc24be66795d5799 Mon Sep 17 00:00:00 2001 From: Andrey Alekseenko Date: Wed, 24 Jan 2024 17:11:09 +0100 Subject: [PATCH 07/11] [urinfo] Fix stray NULLs in stdout We were putting null-terminated strings to std::string data, which made the terminator part of the string content. --- tools/urinfo/utils.hpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/urinfo/utils.hpp b/tools/urinfo/utils.hpp index d2e26255c2..a66b1de657 100644 --- a/tools/urinfo/utils.hpp +++ b/tools/urinfo/utils.hpp @@ -76,6 +76,7 @@ inline std::string getPlatformName(ur_platform_handle_t platform) { std::string name(nameSize, '\0'); UR_CHECK(urPlatformGetInfo(platform, UR_PLATFORM_INFO_NAME, nameSize, name.data(), &nameSize)); + name.pop_back(); // std::string does not need a terminating NULL, remove it here return name; } @@ -86,6 +87,7 @@ inline std::string getDeviceName(ur_device_handle_t device) { std::string name(nameSize, '\0'); UR_CHECK(urDeviceGetInfo(device, UR_DEVICE_INFO_NAME, nameSize, name.data(), &nameSize)); + name.pop_back(); // std::string does not need a terminating NULL, remove it here return name; } @@ -96,6 +98,7 @@ inline std::string getDeviceVersion(ur_device_handle_t device) { std::string name(versionSize, '\0'); UR_CHECK(urDeviceGetInfo(device, UR_DEVICE_INFO_VERSION, versionSize, name.data(), &versionSize)); + name.pop_back(); // std::string does not need a terminating NULL, remove it here return name; } @@ -107,6 +110,7 @@ inline std::string getDeviceDriverVersion(ur_device_handle_t device) { UR_CHECK(urDeviceGetInfo(device, UR_DEVICE_INFO_DRIVER_VERSION, driverVersionSize, name.data(), &driverVersionSize)); + name.pop_back(); // std::string does not need a terminating NULL, remove it here return name; } @@ -136,6 +140,7 @@ printLoaderConfigInfo(ur_loader_config_handle_t loaderConfig, std::string str(size, '\0'); UR_CHECK_WEAK( urLoaderConfigGetInfo(loaderConfig, info, size, str.data(), nullptr)); + str.pop_back(); // std::string does not need a terminating NULL, remove it here std::cout << str << "\n"; } @@ -179,6 +184,7 @@ inline void printPlatformInfo(ur_platform_handle_t platform, UR_CHECK_WEAK(urPlatformGetInfo(platform, info, 0, nullptr, &size)); std::string str(size, '\0'); UR_CHECK_WEAK(urPlatformGetInfo(platform, info, size, str.data(), nullptr)); + str.pop_back(); // std::string does not need a terminating NULL, remove it here std::cout << str << "\n"; } @@ -235,6 +241,7 @@ inline void printDeviceInfo(ur_device_handle_t device, UR_CHECK_WEAK(urDeviceGetInfo(device, info, 0, nullptr, &size)); std::string str(size, 0); UR_CHECK_WEAK(urDeviceGetInfo(device, info, size, str.data(), nullptr)); + str.pop_back(); // std::string does not need a terminating NULL, remove it here std::cout << str << "\n"; } } // namespace urinfo From bdd31cc23b37c4a66b63ee6854ab9bb1601dc38e Mon Sep 17 00:00:00 2001 From: "He, Wenju" Date: Wed, 28 Feb 2024 19:41:43 -0800 Subject: [PATCH 08/11] MipmapFree, createUrMemFromZeImage, urMemRelease --- source/adapters/level_zero/image.cpp | 75 +++++++-------------------- source/adapters/level_zero/memory.cpp | 63 ++++++++-------------- source/adapters/level_zero/memory.hpp | 17 ++++++ 3 files changed, 57 insertions(+), 98 deletions(-) diff --git a/source/adapters/level_zero/image.cpp b/source/adapters/level_zero/image.cpp index 251ddf78b6..77ccf22ef3 100644 --- a/source/adapters/level_zero/image.cpp +++ b/source/adapters/level_zero/image.cpp @@ -595,10 +595,9 @@ urBindlessImagesSampledImageHandleDestroyExp(ur_context_handle_t hContext, ur_device_handle_t hDevice, ur_exp_image_handle_t hImage) { // Sampled image is a combination of unsampled image and sampler. - UR_CALL(urBindlessImagesUnsampledImageHandleDestroyExp(hContext, hDevice, - hImage)); - - return UR_RESULT_SUCCESS; + // Sampler is released in urSamplerRelease. + return urBindlessImagesUnsampledImageHandleDestroyExp(hContext, hDevice, + hImage); } UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesImageAllocateExp( @@ -621,15 +620,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesImageAllocateExp( (hContext->ZeContext, hDevice->ZeDevice, &ZeImageDesc, &ZeImage)); ZE2UR_CALL(zeContextMakeImageResident, (hContext->ZeContext, hDevice->ZeDevice, ZeImage)); - try { - auto UrImage = new _ur_image(hContext, ZeImage, /*OwnZeMemHandle*/ true); - UrImage->ZeImageDesc = ZeImageDesc; - *phImageMem = reinterpret_cast(UrImage); - } catch (const std::bad_alloc &) { - return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } catch (...) { - return UR_RESULT_ERROR_UNKNOWN; - } + UR_CALL(createUrMemFromZeImage(hContext, ZeImage, /*OwnZeMemHandle*/ true, + ZeImageDesc, phImageMem)); return UR_RESULT_SUCCESS; } @@ -638,17 +630,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesImageFreeExp( ur_exp_image_mem_handle_t hImageMem) { std::ignore = hContext; std::ignore = hDevice; - auto *UrImage = reinterpret_cast<_ur_image *>(hImageMem); - if (!UrImage->RefCount.decrementAndTest()) - return UR_RESULT_SUCCESS; - - if (UrImage->OwnNativeHandle) { - auto ZeResult = ZE_CALL_NOCHECK(zeImageDestroy, (UrImage->ZeImage)); - // Gracefully handle the case that L0 was already unloaded. - if (ZeResult && ZeResult != ZE_RESULT_ERROR_UNINITIALIZED) - return ze2urResult(ZeResult); - } - delete UrImage; + UR_CALL(urMemRelease(reinterpret_cast(hImageMem))); return UR_RESULT_SUCCESS; } @@ -657,42 +639,34 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesUnsampledImageCreateExp( ur_exp_image_mem_handle_t hImageMem, const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, ur_mem_handle_t *phMem, ur_exp_image_handle_t *phImage) { - std::ignore = phMem; std::shared_lock Lock(hContext->Mutex); ZeStruct ZeImageDesc; UR_CALL(ur2zeImageDesc(pImageFormat, pImageDesc, ZeImageDesc)); - _ur_image *UrImage = nullptr; + ze_image_handle_t ZeImage; ze_memory_allocation_properties_t MemAllocProperties{ ZE_STRUCTURE_TYPE_MEMORY_ALLOCATION_PROPERTIES}; ZE2UR_CALL(zeMemGetAllocProperties, (hContext->ZeContext, hImageMem, &MemAllocProperties, nullptr)); if (MemAllocProperties.type == ZE_MEMORY_TYPE_UNKNOWN) { - UrImage = reinterpret_cast<_ur_image *>(hImageMem); + _ur_image *UrImage = reinterpret_cast<_ur_image *>(hImageMem); if (!isSameImageDesc(&UrImage->ZeImageDesc, &ZeImageDesc)) { ze_image_bindless_exp_desc_t ZeImageBindlessDesc; ZeImageBindlessDesc.stype = ZE_STRUCTURE_TYPE_BINDLESS_IMAGE_EXP_DESC; ZeImageBindlessDesc.pNext = nullptr; ZeImageBindlessDesc.flags = ZE_IMAGE_BINDLESS_EXP_FLAG_BINDLESS; ZeImageDesc.pNext = &ZeImageBindlessDesc; - ze_image_handle_t ZeImageView; ZE2UR_CALL(zeImageViewCreateExt, (hContext->ZeContext, hDevice->ZeDevice, &ZeImageDesc, - UrImage->ZeImage, &ZeImageView)); + UrImage->ZeImage, &ZeImage)); ZE2UR_CALL(zeContextMakeImageResident, - (hContext->ZeContext, hDevice->ZeDevice, ZeImageView)); - try { - UrImage = new _ur_image(hContext, ZeImageView, /*OwnZeMemHandle*/ true); - UrImage->ZeImageDesc = ZeImageDesc; - *phMem = UrImage; - } catch (const std::bad_alloc &) { - return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } catch (...) { - return UR_RESULT_ERROR_UNKNOWN; - } + (hContext->ZeContext, hDevice->ZeDevice, ZeImage)); + UR_CALL(createUrMemFromZeImage(hContext, ZeImage, /*OwnZeMemHandle*/ true, + ZeImageDesc, phMem)); } else { + ZeImage = UrImage->ZeImage; *phMem = nullptr; } } else { @@ -708,20 +682,12 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesUnsampledImageCreateExp( ZeImageDesc.pNext = &BindlessDesc; - ze_image_handle_t ZeImage; ZE2UR_CALL(zeImageCreate, (hContext->ZeContext, hDevice->ZeDevice, &ZeImageDesc, &ZeImage)); ZE2UR_CALL(zeContextMakeImageResident, (hContext->ZeContext, hDevice->ZeDevice, ZeImage)); - try { - UrImage = new _ur_image(hContext, ZeImage, /*OwnZeMemHandle*/ true); - UrImage->ZeImageDesc = ZeImageDesc; - *phMem = UrImage; - } catch (const std::bad_alloc &) { - return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } catch (...) { - return UR_RESULT_ERROR_UNKNOWN; - } + UR_CALL(createUrMemFromZeImage(hContext, ZeImage, /*OwnZeMemHandle*/ true, + ZeImageDesc, phMem)); } static std::once_flag InitFlag; @@ -738,9 +704,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesUnsampledImageCreateExp( if (!zeImageGetDeviceOffsetExpFunctionPtr) return UR_RESULT_ERROR_INVALID_OPERATION; - uint64_t DeviceOffset; - ZE2UR_CALL(zeImageGetDeviceOffsetExpFunctionPtr, - (UrImage->ZeImage, &DeviceOffset)); + uint64_t DeviceOffset{}; + ZE2UR_CALL(zeImageGetDeviceOffsetExpFunctionPtr, (ZeImage, &DeviceOffset)); *phImage = reinterpret_cast(DeviceOffset); return UR_RESULT_SUCCESS; @@ -936,11 +901,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesMipmapGetLevelExp( UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesMipmapFreeExp( ur_context_handle_t hContext, ur_device_handle_t hDevice, ur_exp_image_mem_handle_t hMem) { - std::ignore = hContext; - std::ignore = hDevice; - std::ignore = hMem; - urPrint("[UR][L0] %s function not implemented!\n", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + return urBindlessImagesImageFreeExp(hContext, hDevice, hMem); } UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesImportOpaqueFDExp( diff --git a/source/adapters/level_zero/memory.cpp b/source/adapters/level_zero/memory.cpp index dc7d60bd2e..0b6ffaaa89 100644 --- a/source/adapters/level_zero/memory.cpp +++ b/source/adapters/level_zero/memory.cpp @@ -1518,30 +1518,21 @@ UR_APIEXPORT ur_result_t UR_APICALL urMemImageCreate( ZE2UR_CALL(zeImageCreate, (Context->ZeContext, Device->ZeDevice, &ZeImageDesc, &ZeImage)); - try { - auto UrImage = new _ur_image(Context, ZeImage, true /*OwnZeMemHandle*/); - *Mem = reinterpret_cast(UrImage); - -#ifndef NDEBUG - UrImage->ZeImageDesc = ZeImageDesc; -#endif // !NDEBUG - - if ((Flags & UR_MEM_FLAG_USE_HOST_POINTER) != 0 || - (Flags & UR_MEM_FLAG_ALLOC_COPY_HOST_POINTER) != 0) { - // Initialize image synchronously with immediate offload. - // zeCommandListAppendImageCopyFromMemory must not be called from - // simultaneous threads with the same command list handle, so we need - // exclusive lock. - std::scoped_lock Lock(Context->ImmediateCommandListMutex); - ZE2UR_CALL(zeCommandListAppendImageCopyFromMemory, - (Context->ZeCommandListInit, ZeImage, Host, nullptr, nullptr, - 0, nullptr)); - } - } catch (const std::bad_alloc &) { - return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } catch (...) { - return UR_RESULT_ERROR_UNKNOWN; + UR_CALL(createUrMemFromZeImage(Context, ZeImage, /*OwnZeMemHandle*/ true, + ZeImageDesc, Mem)); + + if ((Flags & UR_MEM_FLAG_USE_HOST_POINTER) != 0 || + (Flags & UR_MEM_FLAG_ALLOC_COPY_HOST_POINTER) != 0) { + // Initialize image synchronously with immediate offload. + // zeCommandListAppendImageCopyFromMemory must not be called from + // simultaneous threads with the same command list handle, so we need + // exclusive lock. + std::scoped_lock Lock(Context->ImmediateCommandListMutex); + ZE2UR_CALL(zeCommandListAppendImageCopyFromMemory, + (Context->ZeCommandListInit, ZeImage, Host, nullptr, nullptr, 0, + nullptr)); } + return UR_RESULT_SUCCESS; } @@ -1560,30 +1551,20 @@ UR_APIEXPORT ur_result_t UR_APICALL urMemImageCreateWithNativeHandle( ze_image_handle_t ZeHImage = ur_cast(NativeMem); - _ur_image *Image = nullptr; - try { - Image = new _ur_image(Context, ZeHImage, Properties->isNativeHandleOwned); - *Mem = reinterpret_cast(Image); - + ZeStruct ZeImageDesc; #ifndef NDEBUG - ZeStruct ZeImageDesc; - ur_result_t Res = ur2zeImageDesc(ImageFormat, ImageDesc, ZeImageDesc); - if (Res != UR_RESULT_SUCCESS) { - delete Image; - *Mem = nullptr; - return Res; - } - Image->ZeImageDesc = ZeImageDesc; + ur_result_t Res = ur2zeImageDesc(ImageFormat, ImageDesc, ZeImageDesc); + if (Res != UR_RESULT_SUCCESS) { + *Mem = nullptr; + return Res; + } #else std::ignore = ImageFormat; std::ignore = ImageDesc; #endif // !NDEBUG - } catch (const std::bad_alloc &) { - return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } catch (...) { - return UR_RESULT_ERROR_UNKNOWN; - } + UR_CALL(createUrMemFromZeImage( + Context, ZeHImage, Properties->isNativeHandleOwned, ZeImageDesc, Mem)); return UR_RESULT_SUCCESS; } diff --git a/source/adapters/level_zero/memory.hpp b/source/adapters/level_zero/memory.hpp index cd8bb42f41..1fb6d6fa31 100644 --- a/source/adapters/level_zero/memory.hpp +++ b/source/adapters/level_zero/memory.hpp @@ -212,3 +212,20 @@ struct _ur_image final : ur_mem_handle_t_ { // Level Zero image handle. ze_image_handle_t ZeImage; }; + +template +ur_result_t +createUrMemFromZeImage(ur_context_handle_t Context, ze_image_handle_t ZeImage, + bool OwnZeMemHandle, + const ZeStruct &ZeImageDesc, T *UrMem) { + try { + auto UrImage = new _ur_image(Context, ZeImage, OwnZeMemHandle); + UrImage->ZeImageDesc = ZeImageDesc; + *UrMem = reinterpret_cast(UrImage); + } catch (const std::bad_alloc &) { + return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } catch (...) { + return UR_RESULT_ERROR_UNKNOWN; + } + return UR_RESULT_SUCCESS; +} From 64a14c85fa32971f93f18f4a9ecc9723bc85fb24 Mon Sep 17 00:00:00 2001 From: "He, Wenju" Date: Thu, 29 Feb 2024 00:52:10 -0800 Subject: [PATCH 09/11] check ZE_MEMORY_TYPE_DEVICE, error check --- source/adapters/level_zero/image.cpp | 31 +++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/source/adapters/level_zero/image.cpp b/source/adapters/level_zero/image.cpp index 77ccf22ef3..6756f3b672 100644 --- a/source/adapters/level_zero/image.cpp +++ b/source/adapters/level_zero/image.cpp @@ -552,6 +552,10 @@ UR_APIEXPORT ur_result_t UR_APICALL urUSMPitchedAllocExp( size_t *pResultPitch) { std::shared_lock Lock(hContext->Mutex); + UR_ASSERT(hContext && hDevice, UR_RESULT_ERROR_INVALID_NULL_HANDLE); + UR_ASSERT(widthInBytes != 0, UR_RESULT_ERROR_INVALID_USM_SIZE); + UR_ASSERT(ppMem && pResultPitch, UR_RESULT_ERROR_INVALID_NULL_POINTER); + static std::once_flag InitFlag; std::call_once(InitFlag, [&]() { ze_driver_handle_t DriverHandle = hContext->getPlatform()->ZeDriver; @@ -606,6 +610,10 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesImageAllocateExp( ur_exp_image_mem_handle_t *phImageMem) { std::shared_lock Lock(hContext->Mutex); + UR_ASSERT(hContext && hDevice, UR_RESULT_ERROR_INVALID_NULL_HANDLE); + UR_ASSERT(pImageFormat && pImageDesc && phImageMem, + UR_RESULT_ERROR_INVALID_NULL_POINTER); + ZeStruct ZeImageDesc; UR_CALL(ur2zeImageDesc(pImageFormat, pImageDesc, ZeImageDesc)); @@ -641,6 +649,11 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesUnsampledImageCreateExp( ur_exp_image_handle_t *phImage) { std::shared_lock Lock(hContext->Mutex); + UR_ASSERT(hContext && hDevice && hImageMem, + UR_RESULT_ERROR_INVALID_NULL_HANDLE); + UR_ASSERT(pImageFormat && pImageDesc && phMem && phImage, + UR_RESULT_ERROR_INVALID_NULL_POINTER); + ZeStruct ZeImageDesc; UR_CALL(ur2zeImageDesc(pImageFormat, pImageDesc, ZeImageDesc)); @@ -669,7 +682,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesUnsampledImageCreateExp( ZeImage = UrImage->ZeImage; *phMem = nullptr; } - } else { + } else if (MemAllocProperties.type == ZE_MEMORY_TYPE_DEVICE) { ze_image_pitched_exp_desc_t PitchedDesc; PitchedDesc.stype = ZE_STRUCTURE_TYPE_PITCHED_IMAGE_EXP_DESC; PitchedDesc.pNext = nullptr; @@ -688,6 +701,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesUnsampledImageCreateExp( (hContext->ZeContext, hDevice->ZeDevice, ZeImage)); UR_CALL(createUrMemFromZeImage(hContext, ZeImage, /*OwnZeMemHandle*/ true, ZeImageDesc, phMem)); + } else { + return UR_RESULT_ERROR_INVALID_VALUE; } static std::once_flag InitFlag; @@ -741,6 +756,15 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesImageCopyExp( ur_rect_region_t hostExtent, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { std::scoped_lock Lock(hQueue->Mutex); + + UR_ASSERT(hQueue, UR_RESULT_ERROR_INVALID_NULL_HANDLE); + UR_ASSERT(pDst && pSrc && pImageFormat && pImageDesc, + UR_RESULT_ERROR_INVALID_NULL_POINTER); + UR_ASSERT(!(UR_EXP_IMAGE_COPY_FLAGS_MASK & imageCopyFlags), + UR_RESULT_ERROR_INVALID_ENUMERATION); + UR_ASSERT(!(pImageDesc && UR_MEM_TYPE_IMAGE1D_BUFFER < pImageDesc->type), + UR_RESULT_ERROR_INVALID_IMAGE_FORMAT_DESCRIPTOR); + ZeStruct ZeImageDesc; UR_CALL(ur2zeImageDesc(pImageFormat, pImageDesc, ZeImageDesc)); @@ -843,6 +867,11 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesImageCopyExp( UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesImageGetInfoExp( ur_exp_image_mem_handle_t hImageMem, ur_image_info_t propName, void *pPropValue, size_t *pPropSizeRet) { + UR_ASSERT(hImageMem, UR_RESULT_ERROR_INVALID_NULL_HANDLE); + UR_ASSERT(UR_IMAGE_INFO_DEPTH >= propName, + UR_RESULT_ERROR_INVALID_ENUMERATION); + UR_ASSERT(pPropValue || pPropSizeRet, UR_RESULT_ERROR_INVALID_NULL_POINTER); + auto *UrImage = reinterpret_cast<_ur_image *>(hImageMem); ze_image_desc_t &Desc = UrImage->ZeImageDesc; switch (propName) { From c6cb19251b1b5328b940c86511acaf29e914415e Mon Sep 17 00:00:00 2001 From: "He, Wenju" Date: Thu, 29 Feb 2024 01:19:56 -0800 Subject: [PATCH 10/11] format --- source/adapters/level_zero/image.cpp | 6 ++++-- source/adapters/level_zero/memory.cpp | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/source/adapters/level_zero/image.cpp b/source/adapters/level_zero/image.cpp index 6756f3b672..0d986de0ad 100644 --- a/source/adapters/level_zero/image.cpp +++ b/source/adapters/level_zero/image.cpp @@ -354,8 +354,10 @@ ur_result_t ur2zeImageDesc(const ur_image_format_t *ImageFormat, ZeImageDesc.type = ZeImageType; ZeImageDesc.format = ZeFormatDesc; ZeImageDesc.width = ur_cast(ImageDesc->width); - ZeImageDesc.height = std::max(ur_cast(ImageDesc->height), (uint64_t)1); - ZeImageDesc.depth = std::max(ur_cast(ImageDesc->depth), (uint64_t)1); + ZeImageDesc.height = + std::max(ur_cast(ImageDesc->height), (uint64_t)1); + ZeImageDesc.depth = + std::max(ur_cast(ImageDesc->depth), (uint64_t)1); ZeImageDesc.arraylevels = ur_cast(ImageDesc->arraySize); ZeImageDesc.miplevels = ImageDesc->numMipLevel; diff --git a/source/adapters/level_zero/memory.cpp b/source/adapters/level_zero/memory.cpp index 0b6ffaaa89..3fb0d79a61 100644 --- a/source/adapters/level_zero/memory.cpp +++ b/source/adapters/level_zero/memory.cpp @@ -1559,8 +1559,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urMemImageCreateWithNativeHandle( return Res; } #else - std::ignore = ImageFormat; - std::ignore = ImageDesc; + std::ignore = ImageFormat; + std::ignore = ImageDesc; #endif // !NDEBUG UR_CALL(createUrMemFromZeImage( From 1b6c9c23e8efe514bfba99c7053a06a7b07f187e Mon Sep 17 00:00:00 2001 From: Luke Drummond Date: Wed, 28 Feb 2024 18:36:37 +0000 Subject: [PATCH 11/11] [test] Fix infinite loop when we run out of input d6930c70bd7 introduced Status.INPUT_END but didn't introduce the means to handle that eventuality. It looks like the intention might have been to exit with success if we've got to the end of input. However, if we run out of input and still have matches pending, I'm treating this as a failure --- cmake/match.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/cmake/match.py b/cmake/match.py index 3f3f4faff3..e138577165 100755 --- a/cmake/match.py +++ b/cmake/match.py @@ -93,6 +93,18 @@ def main(): print_incorrect_match(match_idx + 1, input_lines[input_idx].strip(), ""); print_content(input_lines, match_lines, ignored_lines) sys.exit(1) + elif status == Status.INPUT_END: + # If we get to the end of the input, but still have pending matches, + # then that's a failure unless all pending matches are optional - + # otherwise we're done + while match_idx < len(match_lines): + if not (match_lines[match_idx].startswith(Tag.OPT.value) or + match_lines[match_idx].startswith(Tag.IGNORE.value)): + print_incorrect_match(match_idx + 1, "", match_lines[match_idx]); + print_content(input_lines, match_lines, ignored_lines) + sys.exit(1) + match_idx += 1 + sys.exit(0) input_line = input_lines[input_idx].strip() if input_idx < len(input_lines) else "" match_line = match_lines[match_idx]