From 4c9fe03bb3f3934ca818f3fab3caec5d5c9a68d7 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Wed, 1 Nov 2023 01:30:44 -0700 Subject: [PATCH 01/72] Add UR Sanitizer Layer --- .gitignore | 2 + CMakeLists.txt | 5 + source/loader/CMakeLists.txt | 12 + .../sanitizer/device_sanitizer_report.hpp | 78 +++ .../sanitizer/sanitizer_interceptor.cpp | 625 ++++++++++++++++++ .../sanitizer/sanitizer_interceptor.hpp | 172 +++++ source/loader/layers/sanitizer/ur_sanddi.cpp | 410 ++++++++++++ .../layers/sanitizer/ur_sanitizer_layer.cpp | 29 + .../layers/sanitizer/ur_sanitizer_layer.hpp | 47 ++ source/loader/ur_lib.hpp | 8 +- 10 files changed, 1387 insertions(+), 1 deletion(-) create mode 100644 source/loader/layers/sanitizer/device_sanitizer_report.hpp create mode 100644 source/loader/layers/sanitizer/sanitizer_interceptor.cpp create mode 100644 source/loader/layers/sanitizer/sanitizer_interceptor.hpp create mode 100644 source/loader/layers/sanitizer/ur_sanddi.cpp create mode 100644 source/loader/layers/sanitizer/ur_sanitizer_layer.cpp create mode 100644 source/loader/layers/sanitizer/ur_sanitizer_layer.hpp diff --git a/.gitignore b/.gitignore index 85770fe15c..2166acc643 100644 --- a/.gitignore +++ b/.gitignore @@ -86,3 +86,5 @@ out/ # External content */**/external + +.cache diff --git a/CMakeLists.txt b/CMakeLists.txt index 0cdd736733..8ef4ef9a08 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,6 +33,7 @@ option(UR_USE_UBSAN "enable UndefinedBehaviorSanitizer" OFF) option(UR_USE_MSAN "enable MemorySanitizer" OFF) option(UR_USE_TSAN "enable ThreadSanitizer" OFF) option(UR_ENABLE_TRACING "enable api tracing through xpti" OFF) +option(UR_ENABLE_SANITIZER "enable device sanitizer" ON) option(UMF_BUILD_SHARED_LIBRARY "Build UMF as shared library" OFF) option(UMF_ENABLE_POOL_TRACKING "Build UMF with pool tracking" OFF) option(UR_BUILD_ADAPTER_L0 "build level 0 adapter from SYCL" OFF) @@ -118,6 +119,10 @@ if(UR_ENABLE_TRACING) endif() endif() +if(UR_ENABLE_SANITIZER) + add_compile_definitions(UR_ENABLE_SANITIZER) +endif() + if(UR_USE_ASAN) add_sanitizer_flag(address) endif() diff --git a/source/loader/CMakeLists.txt b/source/loader/CMakeLists.txt index db796612ea..20b93775f5 100644 --- a/source/loader/CMakeLists.txt +++ b/source/loader/CMakeLists.txt @@ -100,6 +100,18 @@ if(UR_ENABLE_TRACING) ) endif() +if(UR_ENABLE_SANITIZER) + target_sources(ur_loader + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/device_sanitizer_report.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/sanitizer_interceptor.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/sanitizer_interceptor.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/ur_sanitizer_layer.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/ur_sanitizer_layer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/ur_sanddi.cpp + ) +endif() + # link validation backtrace dependencies if(UNIX) diff --git a/source/loader/layers/sanitizer/device_sanitizer_report.hpp b/source/loader/layers/sanitizer/device_sanitizer_report.hpp new file mode 100644 index 0000000000..fd31a61e82 --- /dev/null +++ b/source/loader/layers/sanitizer/device_sanitizer_report.hpp @@ -0,0 +1,78 @@ +#pragma once + +#include + +enum class DeviceSanitizerErrorType : int32_t { + OUT_OF_BOUND, + MISALIGNED, + USE_AFTER_FREE, + OUT_OF_SHADOW_BOUND, + UNKNOWN +}; + +enum class DeviceSanitizerMemoryType : int32_t { + USM_DEVICE, + USM_HOST, + USM_SHARED, + LOCAL, + PRIVATE, + UNKNOWN +}; + +// NOTE Layout of this structure should be aligned with the one in +// sycl/include/sycl/detail/device_sanitizer_report.hpp +struct DeviceSanitizerReport { + int Flag = 0; + + char File[256 + 1] = ""; + char Func[128 + 1] = ""; + + int32_t Line = 0; + + uint64_t GID0 = 0; + uint64_t GID1 = 0; + uint64_t GID2 = 0; + + uint64_t LID0 = 0; + uint64_t LID1 = 0; + uint64_t LID2 = 0; + + bool IsWrite = false; + uint32_t AccessSize = 0; + DeviceSanitizerMemoryType MemoryType; + DeviceSanitizerErrorType ErrorType; + + bool IsRecover = false; +}; + +const char *DeviceSanitizerFormat(DeviceSanitizerMemoryType MemoryType) { + switch (MemoryType) { + case DeviceSanitizerMemoryType::USM_DEVICE: + return "USM Device Memory"; + case DeviceSanitizerMemoryType::USM_HOST: + return "USM Host Memory"; + case DeviceSanitizerMemoryType::USM_SHARED: + return "USM Shared Memory"; + case DeviceSanitizerMemoryType::LOCAL: + return "Local Memory"; + case DeviceSanitizerMemoryType::PRIVATE: + return "Private Memory"; + default: + return "Unknown Memory"; + } +} + +const char *DeviceSanitizerFormat(DeviceSanitizerErrorType ErrorType) { + switch (ErrorType) { + case DeviceSanitizerErrorType::OUT_OF_BOUND: + return "out-of-bound-access"; + case DeviceSanitizerErrorType::MISALIGNED: + return "misaligned-access"; + case DeviceSanitizerErrorType::USE_AFTER_FREE: + return "use-after-free"; + case DeviceSanitizerErrorType::OUT_OF_SHADOW_BOUND: + return "out-of-shadow-bound-access"; + default: + return "unknown-error"; + } +} diff --git a/source/loader/layers/sanitizer/sanitizer_interceptor.cpp b/source/loader/layers/sanitizer/sanitizer_interceptor.cpp new file mode 100644 index 0000000000..5b9a548842 --- /dev/null +++ b/source/loader/layers/sanitizer/sanitizer_interceptor.cpp @@ -0,0 +1,625 @@ +//==---------- sanitizer_interceptor.cpp - Sanitizer interceptor -----------==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "sanitizer_interceptor.hpp" +#include "device_sanitizer_report.hpp" + +#include +#include +#include +#include +#include + +#define ASAN_SHADOW_SCALE 3 +#define ASAN_SHADOW_GRANULARITY (1ULL << ASAN_SHADOW_SCALE) + +// These magic values are written to shadow for better error +// reporting. +const int kUsmDeviceRedzoneMagic = 0x81; +const int kUsmHostRedzoneMagic = 0x82; +const int kUsmSharedRedzoneMagic = 0x83; +const int kUsmDeviceDeallocatedMagic = 0x84; +const int kUsmHostDeallocatedMagic = 0x85; +const int kUsmSharedDeallocatedMagic = 0x86; + +// Same with Asan Stack +const int kPrivateLeftRedzoneMagic = 0xf1; +const int kPrivateMidRedzoneMagic = 0xf2; +const int kPrivateRightRedzoneMagic = 0xf3; + +// These magic values are written to shadow for better error +// reporting. +// const int kAsanHeapLeftRedzoneMagic = 0xfa; +// const int kAsanHeapFreeMagic = 0xfd; +// const int kAsanStackLeftRedzoneMagic = 0xf1; +// const int kAsanStackMidRedzoneMagic = 0xf2; +// const int kAsanStackRightRedzoneMagic = 0xf3; +// const int kAsanStackAfterReturnMagic = 0xf5; +// const int kAsanInitializationOrderMagic = 0xf6; +// const int kAsanUserPoisonedMemoryMagic = 0xf7; +// const int kAsanContiguousContainerOOBMagic = 0xfc; +// const int kAsanStackUseAfterScopeMagic = 0xf8; +// const int kAsanGlobalRedzoneMagic = 0xf9; +// const int kAsanInternalHeapMagic = 0xfe; +// const int kAsanArrayCookieMagic = 0xac; +// const int kAsanIntraObjectRedzone = 0xbb; +// const int kAsanAllocaLeftMagic = 0xca; +// const int kAsanAllocaRightMagic = 0xcb; + +const auto kSPIR_AsanShadowMemoryGlobalStart = "__AsanShadowMemoryGlobalStart"; +const auto kSPIR_AsanShadowMemoryGlobalEnd = "__AsanShadowMemoryGlobalEnd"; + +const auto kSPIR_DeviceSanitizerReportMem = "__DeviceSanitizerReportMem"; + +namespace { + +DeviceSanitizerReport SPIR_DeviceSanitizerReportMem; + +inline constexpr bool IsPowerOfTwo(uptr x) { + return (x & (x - 1)) == 0 && x != 0; +} + +inline constexpr uptr RoundUpTo(uptr Size, uptr boundary) { + assert(IsPowerOfTwo(boundary)); + return (Size + boundary - 1) & ~(boundary - 1); +} + +inline constexpr uptr RoundDownTo(uptr x, uptr boundary) { + assert(IsPowerOfTwo(boundary)); + return x & ~(boundary - 1); +} + +inline constexpr bool IsAligned(uptr a, uptr alignment) { + return (a & (alignment - 1)) == 0; +} + +/* +inline uptr LeastSignificantSetBitIndex(uptr x) { + // CHECK_NE(x, 0U); + unsigned long up; +#if !SANITIZER_WINDOWS || defined(__clang__) || defined(__GNUC__) +#ifdef _WIN64 + up = __builtin_ctzll(x); +#else + up = __builtin_ctzl(x); +#endif +#elif defined(_WIN64) + _BitScanForward64(&up, x); +#else + _BitScanForward(&up, x); +#endif + return up; +} + +inline uptr Log2(uptr x) { + // CHECK(IsPowerOfTwo(x)); + return LeastSignificantSetBitIndex(x); +} +*/ + +// Valid redzone sizes are 16, 32, 64, ... 2048, so we encode them in 3 bits. +// We use adaptive redzones: for larger allocation larger redzones are used. +static u32 RZLog2Size(u32 rz_log) { + // CHECK_LT(rz_log, 8); + return 16 << rz_log; +} + +/* +static u32 RZSize2Log(u32 rz_size) { + // CHECK_GE(rz_size, 16); + // CHECK_LE(rz_size, 2048); + // CHECK(IsPowerOfTwo(rz_size)); + u32 res = Log2(rz_size) - 4; + // CHECK_EQ(rz_size, RZLog2Size(res)); + return res; +} +*/ + +uptr ComputeRZLog(uptr user_requested_size) { + u32 rz_log = user_requested_size <= 64 - 16 ? 0 + : user_requested_size <= 128 - 32 ? 1 + : user_requested_size <= 512 - 64 ? 2 + : user_requested_size <= 4096 - 128 ? 3 + : user_requested_size <= (1 << 14) - 256 ? 4 + : user_requested_size <= (1 << 15) - 512 ? 5 + : user_requested_size <= (1 << 16) - 1024 ? 6 + : 7; + // u32 hdr_log = RZSize2Log(RoundUpToPowerOfTwo(sizeof(ChunkHeader))); + // u32 min_log = RZSize2Log(atomic_load(&min_redzone, memory_order_acquire)); + // u32 max_log = RZSize2Log(atomic_load(&max_redzone, memory_order_acquire)); + // return Min(Max(rz_log, Max(min_log, hdr_log)), Max(max_log, hdr_log)); + return rz_log; +} + +inline constexpr uptr MemToShadow(uptr Addr, uptr ShadowOffset) { + return ShadowOffset + ((Addr) >> ASAN_SHADOW_SCALE); +} + +} // namespace + +ur_result_t SanitizerInterceptor::allocateMemory( + ur_context_handle_t Context, ur_device_handle_t Device, + const ur_usm_desc_t *Properties, ur_usm_pool_handle_t Pool, size_t Size, + void **ResultPtr, USMMemoryType Type) { + (void)Context; + + auto Alignment = Properties->align; + assert(Alignment == 0 || IsPowerOfTwo(Alignment)); + + auto &ContextInfo = getContextInfo(Context); + if (!ContextInfo.Init) { + initContext(Context); + } + + if (Device) { + auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); + if (!DeviceInfo.Init) { + initDevice(Context, Device); + } + if (Alignment == 0) { + Alignment = DeviceInfo.Alignment; + } + } + + if (Alignment == 0) { + // FIXME: OS Defined? + Alignment = 8; + } + + // Calcuate Size + RZSize + uptr rz_log = ComputeRZLog(Size); + uptr rz_size = RZLog2Size(rz_log); + uptr rounded_size = RoundUpTo(Size, Alignment); + uptr NeededSize = rounded_size + rz_size * 2; + + std::cerr << "allocateMemory:" + << "\n user_size: " << Size << "\n rz_size: " << rz_size + << "\n rounded_size: " << rounded_size + << "\n NeededSize: " << NeededSize << std::endl; + + void *Allocated = nullptr; + ur_result_t Result = UR_RESULT_SUCCESS; + if (Type == USMMemoryType::DEVICE) { + Result = m_Dditable.USM.pfnDeviceAlloc(Context, Device, Properties, + Pool, NeededSize, &Allocated); + } else if (Type == USMMemoryType::HOST) { + Result = m_Dditable.USM.pfnHostAlloc(Context, Properties, Pool, + NeededSize, &Allocated); + } else if (Type == USMMemoryType::SHARE) { + Result = m_Dditable.USM.pfnSharedAlloc(Context, Device, Properties, + Pool, NeededSize, &Allocated); + } else { + assert(false && "SanitizerInterceptor::allocateMemory not implemented"); + } + if (Result != UR_RESULT_SUCCESS) { + return Result; + } + + // Enqueue Shadow Memory Init + uptr AllocBegin = reinterpret_cast(Allocated); + uptr AllocEnd = AllocBegin + NeededSize; + uptr UserBegin = AllocBegin + rz_size; + if (!IsAligned(UserBegin, Alignment)) { + UserBegin = RoundUpTo(UserBegin, Alignment); + } + uptr UserEnd = UserBegin + Size; + assert(UserEnd <= AllocEnd); + + *ResultPtr = reinterpret_cast(UserBegin); + + auto MemoryInfo = + AllocatedMemoryInfo{AllocBegin, UserBegin, UserEnd, NeededSize, Type}; + if (Device) { + MemoryInfo.Devices.emplace(Device); + } + + // Update Shadow Memory + if (Device) { + auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); + std::lock_guard Guard(DeviceInfo.Mutex); + DeviceInfo.AllocInfos.emplace_back(MemoryInfo); + } else { + std::lock_guard Guard(ContextInfo.Mutex); + ContextInfo.AllocHostInfos.emplace_back(MemoryInfo); + } + + // Save into AllocatedAddressesMap for releasing + { + std::lock_guard Guard(ContextInfo.Mutex); + ContextInfo.AllocatedAddressesMap[AllocBegin] = MemoryInfo; + std::cout << "AllocatedAddressesMap: " << (void *)AllocBegin << "\n"; + } + + std::cout << "AllocInfos: " << (void *)AllocBegin << " " + << (void *)UserBegin << "-" << (void *)UserEnd << " " + << NeededSize << " " << (void *)Type << std::endl; + + return UR_RESULT_SUCCESS; +} + +ur_result_t SanitizerInterceptor::releaseMemory(ur_context_handle_t Context, + void *Ptr) { + auto &ContextInfo = getContextInfo(Context); + assert(ContextInfo.Init); + + std::cerr << "releaseMemory: " << Ptr << "\n"; + + std::lock_guard Guard(ContextInfo.Mutex); + auto Addr = (uptr)Ptr; + // Find the last element is not greater than key + auto AddressInfoIt = + ContextInfo.AllocatedAddressesMap.upper_bound((uptr)Addr); + if (AddressInfoIt == ContextInfo.AllocatedAddressesMap.begin()) { + std::cerr << "ERROR: releaseMemory failed! AllocatedAddressesMap\n"; + return UR_RESULT_SUCCESS; + } + --AddressInfoIt; + auto &AddressInfo = AddressInfoIt->second; + std::cerr << "AddressInfo: " << AddressInfo.AllocBegin << " " + << AddressInfo.UserBegin << "\n"; + if (Addr != AddressInfo.UserBegin) { + std::cerr << "ERROR: releaseMemory failed! UserBegin\n"; + return UR_RESULT_SUCCESS; + } + + // TODO: Update shadow memory + return m_Dditable.USM.pfnFree(Context, (void *)AddressInfo.AllocBegin); +} + +void SanitizerInterceptor::addQueue(ur_context_handle_t Context, + ur_device_handle_t Device, + ur_queue_handle_t Queue) { + auto &QueueInfo = getQueueInfo(Queue); + QueueInfo.Device = Device; + QueueInfo.Context = Context; +} + +void SanitizerInterceptor::addKernel(ur_program_handle_t Program, + ur_kernel_handle_t Kernel) { + auto &KernelInfo = getKernelInfo(Kernel); + KernelInfo.Program = Program; +} + +bool SanitizerInterceptor::launchKernel(ur_kernel_handle_t Kernel, + ur_queue_handle_t Queue, + ur_event_handle_t &Event) { + // KernelInfo &KernelInfo = getKernelInfo(Kernel); + initKernel(Queue, Kernel); + + updateShadowMemory(Queue, Kernel); + + auto &QueueInfo = getQueueInfo(Queue); + std::lock_guard Guard(QueueInfo.Mutex); + Event = QueueInfo.LastEvent; + QueueInfo.LastEvent = nullptr; + return true; +} + +static void checkSanitizerReport(const char *KernelName) { + auto AH = &SPIR_DeviceSanitizerReportMem; + if (!AH->Flag) { + return; + } + + const char *File = AH->File[0] ? AH->File : ""; + const char *Func = AH->Func[0] ? AH->Func : ""; + + fprintf(stderr, "\n====ERROR: DeviceSanitizer: %s on %s\n\n", + DeviceSanitizerFormat(AH->ErrorType), + DeviceSanitizerFormat(AH->MemoryType)); + fprintf(stderr, + "%s of size %u at kernel <%s> LID(%lu, %lu, %lu) GID(%lu, " + "%lu, %lu)\n", + AH->IsWrite ? "WRITE" : "READ", AH->AccessSize, KernelName, + AH->LID0, AH->LID1, AH->LID2, AH->GID0, AH->GID1, AH->GID2); + fprintf(stderr, " #0 %s %s:%d\n", Func, File, AH->Line); + fflush(stderr); + if (!AH->IsRecover) { + abort(); + } +} + +void SanitizerInterceptor::postLaunchKernel(ur_kernel_handle_t Kernel, + ur_queue_handle_t Queue, + ur_event_handle_t *Event, + bool SetCallback) { + // auto &KernelInfo = getKernelInfo(Kernel); + // auto Program = KernelInfo.Program; + + // ur_event_handle_t ReadEvent{}; + + // If kernel has defined SPIR_DeviceSanitizerReportMem, then we try to read it + // to host, but it's okay that it isn't defined + // auto Ret = m_Dditable.Enqueue.pfnDeviceGlobalVariableRead( + // Queue, Program, kSPIR_DeviceSanitizerReportMem, false, + // sizeof(SPIR_DeviceSanitizerReportMem), 0, + // &SPIR_DeviceSanitizerReportMem, 1, Event, &ReadEvent); + + // if (Ret == UR_RESULT_SUCCESS && SetCallback) { + // Ret = piEventSetCallback( + // ReadEvent, ur_event_handle_t_COMPLETE, + // [](ur_event_handle_t Event, pi_int32 EventCommandStatus, + // void *user_data) { + // (void)Event; + // (void)EventCommandStatus; + // auto *KernelName = (const char *)user_data; + // checkSanitizerReport(KernelName); + // }, + // (void *)KernelInfo.Name.c_str()); + // assert(Ret == UR_RESULT_SUCCESS); + // } + + // *Event = ReadEvent; + return; +} + +std::string SanitizerInterceptor::getKernelName(ur_kernel_handle_t Kernel) { + size_t KernelNameSize = 0; + auto Res = m_Dditable.Kernel.pfnGetInfo( + Kernel, UR_KERNEL_INFO_FUNCTION_NAME, 0, nullptr, &KernelNameSize); + assert(Res == UR_RESULT_SUCCESS); + + std::vector KernelNameBuf(KernelNameSize + 1); + Res = m_Dditable.Kernel.pfnGetInfo(Kernel, UR_KERNEL_INFO_FUNCTION_NAME, + KernelNameSize, KernelNameBuf.data(), + nullptr); + assert(Res == UR_RESULT_SUCCESS); + KernelNameBuf[KernelNameSize] = '\0'; + + return std::string(KernelNameBuf.data(), KernelNameSize); +} + +void SanitizerInterceptor::checkSanitizerError(ur_kernel_handle_t Kernel) { + std::string KernelName = getKernelName(Kernel); + checkSanitizerReport(KernelName.c_str()); +} + +bool SanitizerInterceptor::updateHostShadowMemory( + ur_context_handle_t Context, AllocatedMemoryInfo AllocInfo) { + auto &ContextInfo = getContextInfo(Context); + auto ShadowOffset = ContextInfo.HostShadowOffset; + + uptr tail_beg = RoundUpTo(AllocInfo.UserEnd, ASAN_SHADOW_GRANULARITY); + uptr tail_end = AllocInfo.AllocBegin + AllocInfo.AllocSize; + // user tail + if (tail_beg != AllocInfo.UserEnd) { + auto Value = AllocInfo.UserEnd - + RoundDownTo(AllocInfo.UserEnd, ASAN_SHADOW_GRANULARITY); + auto ShadowPtr = (u8 *)MemToShadow(AllocInfo.UserEnd, ShadowOffset); + *ShadowPtr = Value; + } + auto ShadowByte = AllocInfo.Type == USMMemoryType::DEVICE + ? kUsmDeviceRedzoneMagic + : kUsmSharedRedzoneMagic; + std::memset((void *)MemToShadow(AllocInfo.AllocBegin, ShadowOffset), + ShadowByte, AllocInfo.UserBegin - AllocInfo.AllocBegin); + std::memset((void *)MemToShadow(tail_beg, ShadowOffset), ShadowByte, + tail_end - tail_beg); + return true; +} + +ur_result_t SanitizerInterceptor::piextMemAllocShadow( + ur_context_handle_t Context, ur_device_handle_t Device, void **ShadowOffset, + size_t *ShadowSize) { + return UR_RESULT_SUCCESS; +} + +ur_result_t SanitizerInterceptor::piextEnqueueMemSetShadow( + ur_context_handle_t Context, ur_device_handle_t Device, + ur_queue_handle_t Queue, void *Addr, size_t Size, uint8_t Value, + size_t NumEvents, const ur_event_handle_t *EventsList, + ur_event_handle_t *OutEvent) { + return UR_RESULT_SUCCESS; +} + +ur_result_t SanitizerInterceptor::enqueuePoisonShadow( + ur_context_handle_t Context, ur_device_handle_t Device, + ur_queue_handle_t Queue, uptr Addr, uptr Size, u8 Value, + ur_event_handle_t DepEvent, ur_event_handle_t *OutEvent) { + uint32_t NumEvents = DepEvent ? 1 : 0; + const ur_event_handle_t *EventsList = DepEvent ? &DepEvent : nullptr; + return piextEnqueueMemSetShadow(Context, Device, Queue, (void *)Addr, Size, + Value, NumEvents, EventsList, OutEvent); +} + +void SanitizerInterceptor::enqueueAllocInfo(ur_context_handle_t Context, + ur_device_handle_t Device, + ur_queue_handle_t Queue, + AllocatedMemoryInfo &AllocInfo, + ur_event_handle_t &LastEvent) { + // Init zero + auto Res = + enqueuePoisonShadow(Context, Device, Queue, AllocInfo.AllocBegin, + AllocInfo.AllocSize, 0, LastEvent, &LastEvent); + assert(Res == UR_RESULT_SUCCESS); + + uptr TailBegin = RoundUpTo(AllocInfo.UserEnd, ASAN_SHADOW_GRANULARITY); + uptr TailEnd = AllocInfo.AllocBegin + AllocInfo.AllocSize; + + // User tail + if (TailBegin != AllocInfo.UserEnd) { + auto Value = AllocInfo.UserEnd - + RoundDownTo(AllocInfo.UserEnd, ASAN_SHADOW_GRANULARITY); + auto Res = + enqueuePoisonShadow(Context, Device, Queue, AllocInfo.UserEnd, 1, + Value, LastEvent, &LastEvent); + assert(Res == UR_RESULT_SUCCESS); + } + + int ShadowByte = 0; + switch (AllocInfo.Type) { + case USMMemoryType::HOST: + ShadowByte = kUsmHostRedzoneMagic; + break; + case USMMemoryType::DEVICE: + ShadowByte = kUsmDeviceRedzoneMagic; + break; + case USMMemoryType::SHARE: + ShadowByte = kUsmSharedRedzoneMagic; + break; + default: + ShadowByte = 0xFF; + assert(false && "Unknow AllocInfo.Type"); + break; + } + + // Left red zone + Res = enqueuePoisonShadow(Context, Device, Queue, AllocInfo.AllocBegin, + AllocInfo.UserBegin - AllocInfo.AllocBegin, + ShadowByte, LastEvent, &LastEvent); + assert(Res == UR_RESULT_SUCCESS); + + // Right red zone + Res = enqueuePoisonShadow(Context, Device, Queue, TailBegin, + TailEnd - TailBegin, ShadowByte, LastEvent, + &LastEvent); + assert(Res == UR_RESULT_SUCCESS); +} + +bool SanitizerInterceptor::updateShadowMemory(ur_queue_handle_t Queue, + ur_kernel_handle_t Kernel) { + (void)Kernel; + auto &QueueInfo = getQueueInfo(Queue); + auto Context = QueueInfo.Context; + auto Device = QueueInfo.Device; + + auto &ContextInfo = getContextInfo(Context); + assert(ContextInfo.Init); + auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); + assert(DeviceInfo.Init); + + std::lock_guard QueueGuard(QueueInfo.Mutex); + std::lock_guard DeviceGuard(DeviceInfo.Mutex); + + ur_event_handle_t LastEvent = QueueInfo.LastEvent; + + // FIXME: Always update host USM, but it'd be better to update host USM + // selectively, or each devices once + for (auto &AllocInfo : ContextInfo.AllocHostInfos) { + if (AllocInfo.Devices.find(Device) == AllocInfo.Devices.end()) { + enqueueAllocInfo(Context, Device, Queue, AllocInfo, LastEvent); + AllocInfo.Devices.emplace(Device); + } + } + + for (auto &AllocInfo : DeviceInfo.AllocInfos) { + enqueueAllocInfo(Context, Device, Queue, AllocInfo, LastEvent); + } + DeviceInfo.AllocInfos.clear(); + + QueueInfo.LastEvent = LastEvent; + + return true; +} + +void SanitizerInterceptor::initContext(ur_context_handle_t Context) { + auto &ContextInfo = getContextInfo(Context); + std::lock_guard Guard(ContextInfo.Mutex); + + if (ContextInfo.Init) { + return; + } + + ContextInfo.Init = true; +} + +void SanitizerInterceptor::initDevice(ur_context_handle_t Context, + ur_device_handle_t Device) { + auto &ContextInfo = getContextInfo(Context); + assert(ContextInfo.Init && "Context not inited"); + + assert(Device); + auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); + std::lock_guard Guard(DeviceInfo.Mutex); + + if (DeviceInfo.Init) { + return; + } + + // Query alignment + auto Result = m_Dditable.Device.pfnGetInfo( + Device, UR_DEVICE_INFO_MEM_BASE_ADDR_ALIGN, + sizeof(DeviceInfo.Alignment), &DeviceInfo.Alignment, nullptr); + assert(Result == UR_RESULT_SUCCESS); + + // Allocate shadow memory + size_t ShadowSize{}; + Result = piextMemAllocShadow( + Context, Device, (void **)&DeviceInfo.ShadowOffset, &ShadowSize); + assert(Result == UR_RESULT_SUCCESS); + DeviceInfo.ShadowOffsetEnd = DeviceInfo.ShadowOffset + ShadowSize - 1; + + std::cout << "Device ShadowOffset: " << (void *)DeviceInfo.ShadowOffset + << " - " << (void *)DeviceInfo.ShadowOffsetEnd << "\n"; + + DeviceInfo.Init = true; +} + +bool SanitizerInterceptor::initKernel(ur_queue_handle_t Queue, + ur_kernel_handle_t Kernel) { + auto &KernelInfo = getKernelInfo(Kernel); + auto Program = KernelInfo.Program; + + auto &QueueInfo = getQueueInfo(Queue); + auto Device = QueueInfo.Device; + auto Context = QueueInfo.Context; + + auto &ContextInfo = getContextInfo(Context); + if (!ContextInfo.Init) { + initContext(Context); + } + + auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); + if (!DeviceInfo.Init) { + initDevice(Context, Device); + } + + // Get kernel name + { + std::lock_guard KernelGuard(KernelInfo.Mutex); + if (KernelInfo.Name.empty()) { + KernelInfo.Name = getKernelName(Kernel); + } + } + + std::lock_guard QueueGuard(QueueInfo.Mutex); + ur_event_handle_t LastEvent = QueueInfo.LastEvent; + + bool Res = true; + do { + // Set global variable to program + auto EnqueueWriteGlobal = [&](const char *Name, const void *Value) { + ur_event_handle_t NewEvent{}; + uint32_t NumEvents = LastEvent ? 1 : 0; + const ur_event_handle_t *EventsList = + LastEvent ? &LastEvent : nullptr; + auto Result = m_Dditable.Enqueue.pfnDeviceGlobalVariableWrite( + Queue, Program, Name, false, sizeof(uptr), 0, Value, NumEvents, + EventsList, &NewEvent); + if (Result != UR_RESULT_SUCCESS) { + std::cerr << "WARNING: Device Global Write Failed [" << Name + << "] " << Result << std::endl; + return false; + } + LastEvent = NewEvent; + return true; + }; + + // Device shadow memory offset + EnqueueWriteGlobal(kSPIR_AsanShadowMemoryGlobalStart, + &DeviceInfo.ShadowOffset); + EnqueueWriteGlobal(kSPIR_AsanShadowMemoryGlobalEnd, + &DeviceInfo.ShadowOffsetEnd); + } while (false); + + assert(Res && "Init Kernel Failed"); + + QueueInfo.LastEvent = LastEvent; + + return Res; +} diff --git a/source/loader/layers/sanitizer/sanitizer_interceptor.hpp b/source/loader/layers/sanitizer/sanitizer_interceptor.hpp new file mode 100644 index 0000000000..5ab9f6a60b --- /dev/null +++ b/source/loader/layers/sanitizer/sanitizer_interceptor.hpp @@ -0,0 +1,172 @@ +//==---------- sanitizer_interceptor.hpp - Sanitizer interceptor -----------==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "ur_ddi.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +typedef uintptr_t uptr; +typedef unsigned char u8; +typedef unsigned int u32; + +enum USMMemoryType { + DEVICE, + SHARE, + HOST, +}; + +class SanitizerInterceptor { + public: + SanitizerInterceptor(ur_dditable_t &dditable) : m_Dditable(dditable) {} + + ur_result_t allocateMemory(ur_context_handle_t Context, + ur_device_handle_t Device, + const ur_usm_desc_t *Properties, + ur_usm_pool_handle_t Pool, size_t Size, + void **ResultPtr, USMMemoryType Type); + ur_result_t releaseMemory(ur_context_handle_t Context, void *Ptr); + void addQueue(ur_context_handle_t Context, ur_device_handle_t Device, + ur_queue_handle_t Queue); + void addKernel(ur_program_handle_t Program, ur_kernel_handle_t Kernel); + bool launchKernel(ur_kernel_handle_t Kernel, ur_queue_handle_t Queue, + ur_event_handle_t &Event); + void postLaunchKernel(ur_kernel_handle_t Kernel, ur_queue_handle_t Queue, + ur_event_handle_t *Event, bool SetCallback = true); + void checkSanitizerError(ur_kernel_handle_t Kernel); + + private: + struct AllocatedMemoryInfo { + uptr AllocBegin; + uptr UserBegin; + uptr UserEnd; + size_t AllocSize; + USMMemoryType Type; + std::unordered_set Devices; + }; + + struct DeviceInfo { + bool Init = false; + bool InitPool = false; + + size_t Alignment; + uptr ShadowOffset; + uptr ShadowOffsetEnd; + + std::vector AllocInfos; + std::mutex Mutex; // Lock Init & InitPool & AllocInfos + }; + + enum class DeviceType { CPU, GPU_PVC, GPU_DG2 }; + + struct ContextInfo { + bool Init = false; + + /// AllocatedMemoryInfo.AllocBegin => AllocatedMemoryInfo + /// + /// Use AllocBegin as key can help to detect underflow pointer + std::map AllocatedAddressesMap; + + std::vector AllocHostInfos; + + /// Each context is able to contain multiple devices + std::unordered_map DeviceMap; + + std::mutex Mutex; // Lock Init and Maps + + uptr HostShadowOffset; + + DeviceInfo &getDeviceInfo(ur_device_handle_t Device) { + std::lock_guard Guard(Mutex); + return DeviceMap[Device]; + } + AllocatedMemoryInfo &getAllocatedMemoryInfo(uptr Address) { + std::lock_guard Guard(Mutex); + return AllocatedAddressesMap[Address]; + } + }; + + struct QueueInfo { + ur_device_handle_t Device; + ur_context_handle_t Context; + ur_event_handle_t LastEvent = nullptr; + std::mutex Mutex; // Lock LastEvent + }; + + struct KernelInfo { + ur_program_handle_t Program = nullptr; + std::string Name; + std::mutex Mutex; // Lock Name + }; + + private: + bool updateShadowMemory(ur_queue_handle_t Queue, ur_kernel_handle_t Kernel); + void enqueueAllocInfo(ur_context_handle_t Context, + ur_device_handle_t Device, ur_queue_handle_t Queue, + AllocatedMemoryInfo &AllocInfo, + ur_event_handle_t &LastEvent); + bool updateHostShadowMemory(ur_context_handle_t Context, + AllocatedMemoryInfo AllocInfo); + /// Initialize Global Variables & Kernel Name at first Launch + bool initKernel(ur_queue_handle_t Queue, ur_kernel_handle_t Kernel); + /// Initialze USM Host Memory Pools + void initContext(ur_context_handle_t Context); + /// Initialze USM Device & Shared Memory Pools, Privte & Local Memory Shadow + /// Pools + void initDevice(ur_context_handle_t Context, ur_device_handle_t Device); + std::string getKernelName(ur_kernel_handle_t Kernel); + ur_result_t piextMemAllocShadow(ur_context_handle_t Context, + ur_device_handle_t Device, + void **ShadowOffset, size_t *ShadowSize); + ur_result_t piextEnqueueMemSetShadow(ur_context_handle_t Context, + ur_device_handle_t Device, + ur_queue_handle_t Queue, void *Addr, + size_t Size, uint8_t Value, + size_t NumEvents, + const ur_event_handle_t *EventsList, + ur_event_handle_t *OutEvent); + ur_result_t enqueuePoisonShadow(ur_context_handle_t Context, + ur_device_handle_t Device, + ur_queue_handle_t Queue, uptr Addr, + uptr Size, u8 Value, + ur_event_handle_t DepEvent, + ur_event_handle_t *OutEvent); + + private: + std::unordered_map m_ContextMap; + std::mutex m_ContextMapMutex; + std::unordered_map m_QueueMap; + std::mutex m_QueueMapMutex; + std::unordered_map m_KernelMap; + std::mutex m_KernelMapMutex; + + ContextInfo &getContextInfo(ur_context_handle_t Context) { + std::lock_guard ContextMapGuard(m_ContextMapMutex); + return m_ContextMap[Context]; + } + QueueInfo &getQueueInfo(ur_queue_handle_t Queue) { + std::lock_guard QueueMapGuard(m_QueueMapMutex); + return m_QueueMap[Queue]; + } + KernelInfo &getKernelInfo(ur_kernel_handle_t Kernel) { + std::lock_guard KernelMapGuard(m_KernelMapMutex); + return m_KernelMap[Kernel]; + } + + ur_dditable_t &m_Dditable; +}; diff --git a/source/loader/layers/sanitizer/ur_sanddi.cpp b/source/loader/layers/sanitizer/ur_sanddi.cpp new file mode 100644 index 0000000000..06f4a941f5 --- /dev/null +++ b/source/loader/layers/sanitizer/ur_sanddi.cpp @@ -0,0 +1,410 @@ +/* + * + * Copyright (C) 2023 Intel Corporation + * + * Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. + * See LICENSE.TXT + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + * @file ur_trcddi.cpp + * + */ + +#include "sanitizer_interceptor.hpp" +#include "ur_sanitizer_layer.hpp" + +#include +#include + +namespace ur_sanitizer_layer { + +// /////////////////////////////////////////////////////////////////////////////// +// /// @brief Intercept function for urUSMHostAlloc +// __urdlllocal ur_result_t UR_APICALL urUSMHostAlloc( +// ur_context_handle_t hContext, ///< [in] handle of the context object +// const ur_usm_desc_t +// *pUSMDesc, ///< [in][optional] USM memory allocation descriptor +// ur_usm_pool_handle_t +// pool, ///< [in][optional] Pointer to a pool created using urUSMPoolCreate +// size_t +// size, ///< [in] size in bytes of the USM memory object to be allocated +// void **ppMem ///< [out] pointer to USM host memory object +// ) { +// auto pfnHostAlloc = context.urDdiTable.USM.pfnHostAlloc; + +// if (nullptr == pfnHostAlloc) { +// return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +// } + +// ur_usm_host_alloc_params_t params = {&hContext, &pUSMDesc, &pool, &size, +// &ppMem}; +// uint64_t instance = context.notify_begin(UR_FUNCTION_USM_HOST_ALLOC, +// "urUSMHostAlloc", ¶ms); + +// ur_result_t result = pfnHostAlloc(hContext, pUSMDesc, pool, size, ppMem); + +// context.notify_end(UR_FUNCTION_USM_HOST_ALLOC, "urUSMHostAlloc", ¶ms, +// &result, instance); + +// return result; +// } + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urUSMDeviceAlloc +__urdlllocal ur_result_t UR_APICALL urUSMDeviceAlloc( + ur_context_handle_t hContext, ///< [in] handle of the context object + ur_device_handle_t hDevice, ///< [in] handle of the device object + const ur_usm_desc_t + *pUSMDesc, ///< [in][optional] USM memory allocation descriptor + ur_usm_pool_handle_t + pool, ///< [in][optional] Pointer to a pool created using urUSMPoolCreate + size_t + size, ///< [in] size in bytes of the USM memory object to be allocated + void **ppMem ///< [out] pointer to USM device memory object +) { + auto pfnDeviceAlloc = context.urDdiTable.USM.pfnDeviceAlloc; + + if (nullptr == pfnDeviceAlloc) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + + return context.interceptor->allocateMemory( + hContext, hDevice, pUSMDesc, pool, size, ppMem, USMMemoryType::DEVICE); +} + +// /////////////////////////////////////////////////////////////////////////////// +// /// @brief Intercept function for urUSMSharedAlloc +// __urdlllocal ur_result_t UR_APICALL urUSMSharedAlloc( +// ur_context_handle_t hContext, ///< [in] handle of the context object +// ur_device_handle_t hDevice, ///< [in] handle of the device object +// const ur_usm_desc_t * +// pUSMDesc, ///< [in][optional] Pointer to USM memory allocation descriptor. +// ur_usm_pool_handle_t +// pool, ///< [in][optional] Pointer to a pool created using urUSMPoolCreate +// size_t +// size, ///< [in] size in bytes of the USM memory object to be allocated +// void **ppMem ///< [out] pointer to USM shared memory object +// ) { +// auto pfnSharedAlloc = context.urDdiTable.USM.pfnSharedAlloc; + +// if (nullptr == pfnSharedAlloc) { +// return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +// } + +// ur_usm_shared_alloc_params_t params = {&hContext, &hDevice, &pUSMDesc, +// &pool, &size, &ppMem}; +// uint64_t instance = context.notify_begin(UR_FUNCTION_USM_SHARED_ALLOC, +// "urUSMSharedAlloc", ¶ms); + +// ur_result_t result = +// pfnSharedAlloc(hContext, hDevice, pUSMDesc, pool, size, ppMem); + +// context.notify_end(UR_FUNCTION_USM_SHARED_ALLOC, "urUSMSharedAlloc", +// ¶ms, &result, instance); + +// return result; +// } + +// /////////////////////////////////////////////////////////////////////////////// +// /// @brief Intercept function for urUSMFree +// __urdlllocal ur_result_t UR_APICALL urUSMFree( +// ur_context_handle_t hContext, ///< [in] handle of the context object +// void *pMem ///< [in] pointer to USM memory object +// ) { +// auto pfnFree = context.urDdiTable.USM.pfnFree; + +// if (nullptr == pfnFree) { +// return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +// } + +// ur_usm_free_params_t params = {&hContext, &pMem}; +// uint64_t instance = +// context.notify_begin(UR_FUNCTION_USM_FREE, "urUSMFree", ¶ms); + +// ur_result_t result = pfnFree(hContext, pMem); + +// context.notify_end(UR_FUNCTION_USM_FREE, "urUSMFree", ¶ms, &result, +// instance); + +// return result; +// } + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urKernelCreate +__urdlllocal ur_result_t UR_APICALL urKernelCreate( + ur_program_handle_t hProgram, ///< [in] handle of the program instance + const char *pKernelName, ///< [in] pointer to null-terminated string. + ur_kernel_handle_t + *phKernel ///< [out] pointer to handle of kernel object created. +) { + auto pfnCreate = context.urDdiTable.Kernel.pfnCreate; + + if (nullptr == pfnCreate) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + + ur_result_t result = pfnCreate(hProgram, pKernelName, phKernel); + if (result == UR_RESULT_SUCCESS) { + context.interceptor->addKernel(hProgram, *phKernel); + } + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urQueueCreate +__urdlllocal ur_result_t UR_APICALL urQueueCreate( + ur_context_handle_t hContext, ///< [in] handle of the context object + ur_device_handle_t hDevice, ///< [in] handle of the device object + const ur_queue_properties_t + *pProperties, ///< [in][optional] pointer to queue creation properties. + ur_queue_handle_t + *phQueue ///< [out] pointer to handle of queue object created +) { + auto pfnCreate = context.urDdiTable.Queue.pfnCreate; + + if (nullptr == pfnCreate) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + + ur_result_t result = pfnCreate(hContext, hDevice, pProperties, phQueue); + if (result == UR_RESULT_SUCCESS) { + context.interceptor->addQueue(hContext, hDevice, *phQueue); + } + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urEnqueueKernelLaunch +__urdlllocal ur_result_t UR_APICALL urEnqueueKernelLaunch( + ur_queue_handle_t hQueue, ///< [in] handle of the queue object + ur_kernel_handle_t hKernel, ///< [in] handle of the kernel object + uint32_t + workDim, ///< [in] number of dimensions, from 1 to 3, to specify the global and + ///< work-group work-items + const size_t * + pGlobalWorkOffset, ///< [in] pointer to an array of workDim unsigned values that specify the + ///< offset used to calculate the global ID of a work-item + const size_t * + pGlobalWorkSize, ///< [in] pointer to an array of workDim unsigned values that specify the + ///< number of global work-items in workDim that will execute the kernel + ///< function + const size_t * + pLocalWorkSize, ///< [in][optional] pointer to an array of workDim unsigned values that + ///< specify the number of local work-items forming a work-group that will + ///< execute the kernel function. + ///< If nullptr, the runtime implementation will choose the work-group + ///< size. + uint32_t numEventsInWaitList, ///< [in] size of the event wait list + const ur_event_handle_t * + phEventWaitList, ///< [in][optional][range(0, numEventsInWaitList)] pointer to a list of + ///< events that must be complete before the kernel execution. + ///< If nullptr, the numEventsInWaitList must be 0, indicating that no wait + ///< event. + ur_event_handle_t * + phEvent ///< [out][optional] return an event object that identifies this particular + ///< kernel execution instance. +) { + auto pfnKernelLaunch = context.urDdiTable.Enqueue.pfnKernelLaunch; + + if (nullptr == pfnKernelLaunch) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + + std::cerr << "=== __asan_piEnqueueKernelLaunch" << std::endl; + ur_event_handle_t lk_event{}; + std::vector events(numEventsInWaitList + 1); + for (unsigned i = 0; i < numEventsInWaitList; ++i) { + events.push_back(phEventWaitList[i]); + } + + // launchKernel must append to num_events_in_wait_list, not prepend + context.interceptor->launchKernel(hKernel, hQueue, lk_event); + if (lk_event) { + events.push_back(lk_event); + } + + ur_result_t result = pfnKernelLaunch( + hQueue, hKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, + pLocalWorkSize, numEventsInWaitList, phEventWaitList, phEvent); + + if (result == UR_RESULT_SUCCESS) { + context.interceptor->postLaunchKernel(hKernel, hQueue, phEvent, false); + } + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Exported function for filling application's Enqueue table +/// with current process' addresses +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// - ::UR_RESULT_ERROR_UNSUPPORTED_VERSION +__urdlllocal ur_result_t UR_APICALL urGetEnqueueProcAddrTable( + ur_api_version_t version, ///< [in] API version requested + ur_enqueue_dditable_t + *pDdiTable ///< [in,out] pointer to table of DDI function pointers +) { + // auto &dditable = ur_sanitizer_layer::context.urDdiTable.Enqueue; + + if (nullptr == pDdiTable) { + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + } + + if (UR_MAJOR_VERSION(ur_sanitizer_layer::context.version) != + UR_MAJOR_VERSION(version) || + UR_MINOR_VERSION(ur_sanitizer_layer::context.version) > + UR_MINOR_VERSION(version)) { + return UR_RESULT_ERROR_UNSUPPORTED_VERSION; + } + + ur_result_t result = UR_RESULT_SUCCESS; + + // dditable.pfnKernelLaunch = pDdiTable->pfnKernelLaunch; + pDdiTable->pfnKernelLaunch = ur_sanitizer_layer::urEnqueueKernelLaunch; + + return result; +} +/////////////////////////////////////////////////////////////////////////////// +/// @brief Exported function for filling application's Kernel table +/// with current process' addresses +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// - ::UR_RESULT_ERROR_UNSUPPORTED_VERSION +__urdlllocal ur_result_t UR_APICALL urGetKernelProcAddrTable( + ur_api_version_t version, ///< [in] API version requested + ur_kernel_dditable_t + *pDdiTable ///< [in,out] pointer to table of DDI function pointers +) { + // auto &dditable = ur_sanitizer_layer::context.urDdiTable.Kernel; + + if (nullptr == pDdiTable) { + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + } + + if (UR_MAJOR_VERSION(ur_sanitizer_layer::context.version) != + UR_MAJOR_VERSION(version) || + UR_MINOR_VERSION(ur_sanitizer_layer::context.version) > + UR_MINOR_VERSION(version)) { + return UR_RESULT_ERROR_UNSUPPORTED_VERSION; + } + + ur_result_t result = UR_RESULT_SUCCESS; + + // dditable.pfnCreate = pDdiTable->pfnCreate; + pDdiTable->pfnCreate = ur_sanitizer_layer::urKernelCreate; + + return result; +} +/////////////////////////////////////////////////////////////////////////////// +/// @brief Exported function for filling application's Queue table +/// with current process' addresses +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// - ::UR_RESULT_ERROR_UNSUPPORTED_VERSION +__urdlllocal ur_result_t UR_APICALL urGetQueueProcAddrTable( + ur_api_version_t version, ///< [in] API version requested + ur_queue_dditable_t + *pDdiTable ///< [in,out] pointer to table of DDI function pointers +) { + // auto &dditable = ur_sanitizer_layer::context.urDdiTable.Queue; + + if (nullptr == pDdiTable) { + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + } + + if (UR_MAJOR_VERSION(ur_sanitizer_layer::context.version) != + UR_MAJOR_VERSION(version) || + UR_MINOR_VERSION(ur_sanitizer_layer::context.version) > + UR_MINOR_VERSION(version)) { + return UR_RESULT_ERROR_UNSUPPORTED_VERSION; + } + + ur_result_t result = UR_RESULT_SUCCESS; + + // dditable.pfnCreate = pDdiTable->pfnCreate; + pDdiTable->pfnCreate = ur_sanitizer_layer::urQueueCreate; + + return result; +} +/////////////////////////////////////////////////////////////////////////////// +/// @brief Exported function for filling application's USM table +/// with current process' addresses +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// - ::UR_RESULT_ERROR_UNSUPPORTED_VERSION +__urdlllocal ur_result_t UR_APICALL urGetUSMProcAddrTable( + ur_api_version_t version, ///< [in] API version requested + ur_usm_dditable_t + *pDdiTable ///< [in,out] pointer to table of DDI function pointers +) { + // auto &dditable = ur_sanitizer_layer::context.urDdiTable.USM; + + if (nullptr == pDdiTable) { + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + } + + if (UR_MAJOR_VERSION(ur_sanitizer_layer::context.version) != + UR_MAJOR_VERSION(version) || + UR_MINOR_VERSION(ur_sanitizer_layer::context.version) > + UR_MINOR_VERSION(version)) { + return UR_RESULT_ERROR_UNSUPPORTED_VERSION; + } + + ur_result_t result = UR_RESULT_SUCCESS; + + // dditable.pfnHostAlloc = pDdiTable->pfnHostAlloc; + // pDdiTable->pfnHostAlloc = ur_sanitizer_layer::urUSMHostAlloc; + + // dditable.pfnDeviceAlloc = pDdiTable->pfnDeviceAlloc; + pDdiTable->pfnDeviceAlloc = ur_sanitizer_layer::urUSMDeviceAlloc; + + // dditable.pfnSharedAlloc = pDdiTable->pfnSharedAlloc; + // pDdiTable->pfnSharedAlloc = ur_sanitizer_layer::urUSMSharedAlloc; + + // dditable.pfnFree = pDdiTable->pfnFree; + // pDdiTable->pfnFree = ur_sanitizer_layer::urUSMFree; + + return result; +} + +ur_result_t context_t::init(ur_dditable_t *dditable, + const std::set &enabledLayerNames) { + ur_result_t result = UR_RESULT_SUCCESS; + + std::cout << "ur_sanitizer_layer context_t::init\n"; + + // if (!enabledLayerNames.count(name)) { + // return result; + // } + + if (UR_RESULT_SUCCESS == result) { + // FIXME: Just copy needed APIs? + urDdiTable = *dditable; + + result = ur_sanitizer_layer::urGetEnqueueProcAddrTable( + UR_API_VERSION_CURRENT, &dditable->Enqueue); + + result = ur_sanitizer_layer::urGetKernelProcAddrTable( + UR_API_VERSION_CURRENT, &dditable->Kernel); + + result = ur_sanitizer_layer::urGetQueueProcAddrTable( + UR_API_VERSION_CURRENT, &dditable->Queue); + + result = ur_sanitizer_layer::urGetUSMProcAddrTable( + UR_API_VERSION_CURRENT, &dditable->USM); + } + + return result; +} +} // namespace ur_sanitizer_layer diff --git a/source/loader/layers/sanitizer/ur_sanitizer_layer.cpp b/source/loader/layers/sanitizer/ur_sanitizer_layer.cpp new file mode 100644 index 0000000000..6cd214e9a9 --- /dev/null +++ b/source/loader/layers/sanitizer/ur_sanitizer_layer.cpp @@ -0,0 +1,29 @@ +/* + * + * Copyright (C) 2023 Intel Corporation + * + * Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. + * See LICENSE.TXT + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + * @file ur_sanitizer_layer.cpp + * + */ +#include "ur_sanitizer_layer.hpp" +#include "sanitizer_interceptor.hpp" +#include "ur_api.h" +#include "ur_util.hpp" + +#include + +namespace ur_sanitizer_layer { +context_t context; + +/////////////////////////////////////////////////////////////////////////////// +context_t::context_t() : interceptor(new SanitizerInterceptor(urDdiTable)) {} + +bool context_t::isAvailable() const { return true; } + +/////////////////////////////////////////////////////////////////////////////// +context_t::~context_t() {} +} // namespace ur_sanitizer_layer diff --git a/source/loader/layers/sanitizer/ur_sanitizer_layer.hpp b/source/loader/layers/sanitizer/ur_sanitizer_layer.hpp new file mode 100644 index 0000000000..fe7aafed06 --- /dev/null +++ b/source/loader/layers/sanitizer/ur_sanitizer_layer.hpp @@ -0,0 +1,47 @@ +/* + * + * Copyright (C) 2023 Corporation + * + * Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. + * See LICENSE.TXT + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + * @file ur_sanitizer_layer.h + * + */ + +#ifndef UR_SANITIZER_LAYER_H +#define UR_SANITIZER_LAYER_H 1 + +#include "ur_ddi.h" +#include "ur_proxy_layer.hpp" +#include "ur_util.hpp" + +#define SANITIZER_COMP_NAME "sanitizer layer" + +class SanitizerInterceptor; + +namespace ur_sanitizer_layer { +/////////////////////////////////////////////////////////////////////////////// +class __urdlllocal context_t : public proxy_layer_context_t { + public: + ur_dditable_t urDdiTable = {}; + SanitizerInterceptor *interceptor = {}; + + context_t(); + ~context_t(); + + bool isAvailable() const override; + + std::vector getNames() const override { return {name}; } + ur_result_t init(ur_dditable_t *dditable, + const std::set &enabledLayerNames) override; + + private: + const std::string name = "UR_LAYER_SANITIZER"; +}; + +extern context_t context; +} // namespace ur_sanitizer_layer + +#endif /* UR_sanitizer_LAYER_H */ diff --git a/source/loader/ur_lib.hpp b/source/loader/ur_lib.hpp index 1f0f23658b..12f3877932 100644 --- a/source/loader/ur_lib.hpp +++ b/source/loader/ur_lib.hpp @@ -22,6 +22,9 @@ #if UR_ENABLE_TRACING #include "tracing/ur_tracing_layer.hpp" #endif +#if UR_ENABLE_SANITIZER +#include "sanitizer/ur_sanitizer_layer.hpp" +#endif #include #include @@ -66,7 +69,10 @@ class __urdlllocal context_t { const std::vector layers = { &ur_validation_layer::context, #if UR_ENABLE_TRACING - &ur_tracing_layer::context + &ur_tracing_layer::context, +#endif +#if UR_ENABLE_SANITIZER + &ur_sanitizer_layer::context #endif }; std::string availableLayers; From d6e589b82b5d5dde442611edae5bbd3168f4c0a6 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Wed, 1 Nov 2023 05:52:40 -0700 Subject: [PATCH 02/72] UR Sanitizer: Add shadow memory alloc & memset --- .../sanitizer/sanitizer_interceptor.cpp | 150 ++++++++++++++++-- .../sanitizer/sanitizer_interceptor.hpp | 4 +- source/loader/layers/sanitizer/ur_sanddi.cpp | 5 +- 3 files changed, 146 insertions(+), 13 deletions(-) diff --git a/source/loader/layers/sanitizer/sanitizer_interceptor.cpp b/source/loader/layers/sanitizer/sanitizer_interceptor.cpp index 5b9a548842..29454b34a2 100644 --- a/source/loader/layers/sanitizer/sanitizer_interceptor.cpp +++ b/source/loader/layers/sanitizer/sanitizer_interceptor.cpp @@ -403,17 +403,144 @@ bool SanitizerInterceptor::updateHostShadowMemory( return true; } -ur_result_t SanitizerInterceptor::piextMemAllocShadow( - ur_context_handle_t Context, ur_device_handle_t Device, void **ShadowOffset, - size_t *ShadowSize) { +uptr MemToShadow_CPU(uptr USM_SHADOW_BASE, uptr UPtr) { + return USM_SHADOW_BASE + (UPtr >> 3); +} + +uptr MemToShadow_PVC(uptr USM_SHADOW_BASE, uptr UPtr) { + if (UPtr & 0xFF00000000000000ULL) { // Device USM + return USM_SHADOW_BASE + 0x200000000000ULL + + ((UPtr & 0xFFFFFFFFFFFFULL) >> 3); + } else { // Only consider 47bit VA + return USM_SHADOW_BASE + ((UPtr & 0x7FFFFFFFFFFFULL) >> 3); + } +} + +uptr MemToShadow_DG2(uptr USM_SHADOW_BASE, uptr UPtr) { + if (UPtr & (~0xFFFFFFFFFFFFULL)) { // Device USM + return USM_SHADOW_BASE + ((UPtr & 0xFFFFFFFFFFFFULL) >> 3); + } else { + return USM_SHADOW_BASE + (UPtr >> 3); + } +} + +ur_result_t +SanitizerInterceptor::piextMemAllocShadow(ur_context_handle_t Context, + ur_device_handle_t Device) { + auto &ContextInfo = getContextInfo(Context); + auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); + if (DeviceInfo.Type == UR_DEVICE_TYPE_CPU) { + DeviceInfo.ShadowOffset = 0x00007fff7fffULL; + DeviceInfo.ShadowOffsetEnd = 0x10007fff7fffULL; + } else if (DeviceInfo.Type == UR_DEVICE_TYPE_GPU) { + /// SHADOW MAPPING (PVC, with CPU 47bit) + /// Host/Shared USM : 0x0 ~ 0x0fff_ffff_ffff + /// ? : 0x1000_0000_0000 ~ 0x1fff_ffff_ffff + /// Device USM : 0x2000_0000_0000 ~ 0x3fff_ffff_ffff + constexpr size_t SHADOW_SIZE = 1ULL << 46; + + // TODO: Protect Bad Zone + auto URes = m_Dditable.VirtualMem.pfnReserve( + Context, nullptr, SHADOW_SIZE, (void **)&DeviceInfo.ShadowOffset); + if (URes != UR_RESULT_SUCCESS) { + printf("urVirtualMemReserve(): %p\n", (void *)URes); + } + assert(URes == UR_RESULT_SUCCESS); + + DeviceInfo.ShadowOffsetEnd = DeviceInfo.ShadowOffset + SHADOW_SIZE; + } else { + assert(false && "Unsupport device type"); + } return UR_RESULT_SUCCESS; } ur_result_t SanitizerInterceptor::piextEnqueueMemSetShadow( ur_context_handle_t Context, ur_device_handle_t Device, - ur_queue_handle_t Queue, void *Addr, size_t Size, uint8_t Value, - size_t NumEvents, const ur_event_handle_t *EventsList, + ur_queue_handle_t Queue, void *Ptr, size_t Size, uint8_t Value, + size_t NumEventsInWaitList, const ur_event_handle_t *EventsWaitList, ur_event_handle_t *OutEvent) { + auto &ContextInfo = getContextInfo(Context); + auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); + if (DeviceInfo.Type == UR_DEVICE_TYPE_CPU) { + assert(false && "Unsupport device type"); + } else if (DeviceInfo.Type == UR_DEVICE_TYPE_GPU) { + const uptr UPtr = (uptr)Ptr; + + ur_event_handle_t InternalEvent{}; + ur_event_handle_t *Event = OutEvent ? OutEvent : &InternalEvent; + + uptr ShadowBegin = MemToShadow_PVC(DeviceInfo.ShadowOffset, UPtr); + uptr ShadowEnd = + MemToShadow_PVC(DeviceInfo.ShadowOffset, UPtr + Size - 1); + + // Maybe in future, we needn't to map physical memory manually + const bool IsNeedMapPhysicalMem = true; + + if (IsNeedMapPhysicalMem) { + // We use fixed GPU PageSize: 64KB + const size_t PageSize = 64 * 1024u; + + ur_physical_mem_properties_t Desc{ + UR_STRUCTURE_TYPE_PHYSICAL_MEM_PROPERTIES, nullptr, 0}; + static ur_physical_mem_handle_t PhysicalMem{}; + + // Make sure [Ptr, Ptr + Size] is mapped to physical memory + for (auto MappedPtr = RoundDownTo(ShadowBegin, PageSize); + MappedPtr <= ShadowEnd; MappedPtr += PageSize) { + if (!PhysicalMem) { + auto URes = m_Dditable.PhysicalMem.pfnCreate( + Context, Device, PageSize, &Desc, &PhysicalMem); + if (URes != UR_RESULT_SUCCESS) { + printf(" zePhysicalMemCreate(): %p\n", (void *)URes); + } + assert(URes == UR_RESULT_SUCCESS); + } + + printf(" zeVirtualMemMap: %p ~ %p\n", MappedPtr, + MappedPtr + PageSize - 1); + + // FIXME: No flag to check the failed reason is VA is already mapped + auto URes = m_Dditable.VirtualMem.pfnMap( + Context, (void *)MappedPtr, PageSize, PhysicalMem, 0, + UR_VIRTUAL_MEM_ACCESS_FLAG_READ_WRITE); + if (URes != UR_RESULT_SUCCESS) { + printf(" zeVirtualMemMap(): %p\n", (void *)URes); + } + + // Initialize to zero + if (URes == UR_RESULT_SUCCESS) { + // Reset PhysicalMem to null since it's been mapped + PhysicalMem = nullptr; + + // FIXME: Maybe we needn't to initialize shadow memory to zero? Or it'd be better to be a negative value? + const char Pattern[] = {0}; + + auto URes = m_Dditable.Enqueue.pfnUSMFill( + Queue, (void *)MappedPtr, 1, Pattern, PageSize, + NumEventsInWaitList, EventsWaitList, Event); + if (URes != UR_RESULT_SUCCESS) { + printf(" urEnqueueUSMFill(): %p\n", (void *)URes); + } + assert(URes == UR_RESULT_SUCCESS); + + NumEventsInWaitList = 1; + EventsWaitList = Event; + } + } + } + + const char Pattern[] = {(char)Value}; + auto URes = m_Dditable.Enqueue.pfnUSMFill( + Queue, (void *)ShadowBegin, 1, Pattern, + (ShadowEnd - ShadowBegin + 1), NumEventsInWaitList, EventsWaitList, + Event); + if (URes != UR_RESULT_SUCCESS) { + printf(" urEnqueueUSMFill(): %p\n", (void *)URes); + } + assert(URes == UR_RESULT_SUCCESS); + } else { + assert(false && "Unsupport device type"); + } return UR_RESULT_SUCCESS; } @@ -541,18 +668,21 @@ void SanitizerInterceptor::initDevice(ur_context_handle_t Context, return; } + // Query Device Type + auto Result = m_Dditable.Device.pfnGetInfo(Device, UR_DEVICE_INFO_TYPE, + sizeof(DeviceInfo.Type), + &DeviceInfo.Type, nullptr); + assert(Result == UR_RESULT_SUCCESS); + // Query alignment - auto Result = m_Dditable.Device.pfnGetInfo( + Result = m_Dditable.Device.pfnGetInfo( Device, UR_DEVICE_INFO_MEM_BASE_ADDR_ALIGN, sizeof(DeviceInfo.Alignment), &DeviceInfo.Alignment, nullptr); assert(Result == UR_RESULT_SUCCESS); // Allocate shadow memory - size_t ShadowSize{}; - Result = piextMemAllocShadow( - Context, Device, (void **)&DeviceInfo.ShadowOffset, &ShadowSize); + Result = piextMemAllocShadow(Context, Device); assert(Result == UR_RESULT_SUCCESS); - DeviceInfo.ShadowOffsetEnd = DeviceInfo.ShadowOffset + ShadowSize - 1; std::cout << "Device ShadowOffset: " << (void *)DeviceInfo.ShadowOffset << " - " << (void *)DeviceInfo.ShadowOffsetEnd << "\n"; diff --git a/source/loader/layers/sanitizer/sanitizer_interceptor.hpp b/source/loader/layers/sanitizer/sanitizer_interceptor.hpp index 5ab9f6a60b..7c15cf153b 100644 --- a/source/loader/layers/sanitizer/sanitizer_interceptor.hpp +++ b/source/loader/layers/sanitizer/sanitizer_interceptor.hpp @@ -64,6 +64,7 @@ class SanitizerInterceptor { bool Init = false; bool InitPool = false; + ur_device_type_t Type; size_t Alignment; uptr ShadowOffset; uptr ShadowOffsetEnd; @@ -131,8 +132,7 @@ class SanitizerInterceptor { void initDevice(ur_context_handle_t Context, ur_device_handle_t Device); std::string getKernelName(ur_kernel_handle_t Kernel); ur_result_t piextMemAllocShadow(ur_context_handle_t Context, - ur_device_handle_t Device, - void **ShadowOffset, size_t *ShadowSize); + ur_device_handle_t Device); ur_result_t piextEnqueueMemSetShadow(ur_context_handle_t Context, ur_device_handle_t Device, ur_queue_handle_t Queue, void *Addr, diff --git a/source/loader/layers/sanitizer/ur_sanddi.cpp b/source/loader/layers/sanitizer/ur_sanddi.cpp index 06f4a941f5..a11b854389 100644 --- a/source/loader/layers/sanitizer/ur_sanddi.cpp +++ b/source/loader/layers/sanitizer/ur_sanddi.cpp @@ -67,6 +67,7 @@ __urdlllocal ur_result_t UR_APICALL urUSMDeviceAlloc( if (nullptr == pfnDeviceAlloc) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + std::cerr << "=== urUSMDeviceAlloc" << std::endl; return context.interceptor->allocateMemory( hContext, hDevice, pUSMDesc, pool, size, ppMem, USMMemoryType::DEVICE); @@ -142,6 +143,7 @@ __urdlllocal ur_result_t UR_APICALL urKernelCreate( if (nullptr == pfnCreate) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + std::cerr << "=== urKernelCreate" << std::endl; ur_result_t result = pfnCreate(hProgram, pKernelName, phKernel); if (result == UR_RESULT_SUCCESS) { @@ -166,6 +168,7 @@ __urdlllocal ur_result_t UR_APICALL urQueueCreate( if (nullptr == pfnCreate) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + std::cerr << "=== urQueueCreate" << std::endl; ur_result_t result = pfnCreate(hContext, hDevice, pProperties, phQueue); if (result == UR_RESULT_SUCCESS) { @@ -212,7 +215,7 @@ __urdlllocal ur_result_t UR_APICALL urEnqueueKernelLaunch( return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } - std::cerr << "=== __asan_piEnqueueKernelLaunch" << std::endl; + std::cerr << "=== urEnqueueKernelLaunch" << std::endl; ur_event_handle_t lk_event{}; std::vector events(numEventsInWaitList + 1); for (unsigned i = 0; i < numEventsInWaitList; ++i) { From 67fcd5e11323ee2e2de88767951e5222661882f9 Mon Sep 17 00:00:00 2001 From: "Larsen, Steffen" Date: Mon, 9 Oct 2023 09:15:31 -0700 Subject: [PATCH 03/72] [UR][CUDA][L0][HIP] Add virtual memory adapter implementations This commit adds the adapter implementations of the virtual memory extension functionality to be used in https://github.com/intel/llvm/pull/8954. Signed-off-by: Larsen, Steffen --- source/adapters/cuda/CMakeLists.txt | 3 + source/adapters/cuda/device.cpp | 2 + source/adapters/cuda/physical_mem.cpp | 58 ++++++++ source/adapters/cuda/physical_mem.hpp | 68 +++++++++ source/adapters/cuda/virtual_mem.cpp | 138 +++++++++++++++++++ source/adapters/hip/CMakeLists.txt | 3 + source/adapters/hip/device.cpp | 2 + source/adapters/hip/physical_mem.cpp | 36 +++++ source/adapters/hip/physical_mem.hpp | 30 ++++ source/adapters/hip/virtual_mem.cpp | 69 ++++++++++ source/adapters/level_zero/CMakeLists.txt | 3 + source/adapters/level_zero/common.cpp | 3 + source/adapters/level_zero/device.cpp | 3 + source/adapters/level_zero/physical_mem.cpp | 54 ++++++++ source/adapters/level_zero/physical_mem.hpp | 24 ++++ source/adapters/level_zero/ur_level_zero.hpp | 1 + source/adapters/level_zero/virtual_mem.cpp | 120 ++++++++++++++++ 17 files changed, 617 insertions(+) create mode 100644 source/adapters/cuda/physical_mem.cpp create mode 100644 source/adapters/cuda/physical_mem.hpp create mode 100644 source/adapters/cuda/virtual_mem.cpp create mode 100644 source/adapters/hip/physical_mem.cpp create mode 100644 source/adapters/hip/physical_mem.hpp create mode 100644 source/adapters/hip/virtual_mem.cpp create mode 100644 source/adapters/level_zero/physical_mem.cpp create mode 100644 source/adapters/level_zero/physical_mem.hpp create mode 100644 source/adapters/level_zero/virtual_mem.cpp diff --git a/source/adapters/cuda/CMakeLists.txt b/source/adapters/cuda/CMakeLists.txt index 4a8c1d1f59..cf97419035 100644 --- a/source/adapters/cuda/CMakeLists.txt +++ b/source/adapters/cuda/CMakeLists.txt @@ -27,6 +27,8 @@ add_ur_adapter(${TARGET_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/kernel.cpp ${CMAKE_CURRENT_SOURCE_DIR}/memory.hpp ${CMAKE_CURRENT_SOURCE_DIR}/memory.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.cpp ${CMAKE_CURRENT_SOURCE_DIR}/platform.hpp ${CMAKE_CURRENT_SOURCE_DIR}/platform.cpp ${CMAKE_CURRENT_SOURCE_DIR}/program.hpp @@ -38,6 +40,7 @@ add_ur_adapter(${TARGET_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/tracing.cpp ${CMAKE_CURRENT_SOURCE_DIR}/usm.cpp ${CMAKE_CURRENT_SOURCE_DIR}/usm_p2p.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/virtual_mem.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../ur/ur.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../ur/ur.hpp ) diff --git a/source/adapters/cuda/device.cpp b/source/adapters/cuda/device.cpp index a4877236ae..c27dbdbb52 100644 --- a/source/adapters/cuda/device.cpp +++ b/source/adapters/cuda/device.cpp @@ -1017,6 +1017,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, // TODO: Investigate if this information is available on CUDA. case UR_DEVICE_INFO_HOST_PIPE_READ_WRITE_SUPPORTED: return ReturnValue(false); + case UR_DEVICE_INFO_VIRTUAL_MEMORY_SUPPORT: + return ReturnValue(true); case UR_DEVICE_INFO_ESIMD_SUPPORT: return ReturnValue(false); case UR_DEVICE_INFO_MAX_READ_WRITE_IMAGE_ARGS: diff --git a/source/adapters/cuda/physical_mem.cpp b/source/adapters/cuda/physical_mem.cpp new file mode 100644 index 0000000000..177bc2234d --- /dev/null +++ b/source/adapters/cuda/physical_mem.cpp @@ -0,0 +1,58 @@ +//===--------- physical_mem.cpp - CUDA Adapter ----------------------------===// +// +// Copyright (C) 2023 Intel Corporation +// +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See LICENSE.TXT +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "physical_mem.hpp" +#include "common.hpp" +#include "context.hpp" +#include "event.hpp" + +#include +#include + +UR_APIEXPORT ur_result_t UR_APICALL urPhysicalMemCreate( + ur_context_handle_t hContext, ur_device_handle_t hDevice, size_t size, + [[maybe_unused]] const ur_physical_mem_properties_t *pProperties, + ur_physical_mem_handle_t *phPhysicalMem) { + CUmemAllocationProp AllocProps = {}; + AllocProps.location.type = CU_MEM_LOCATION_TYPE_DEVICE; + AllocProps.type = CU_MEM_ALLOCATION_TYPE_PINNED; + UR_ASSERT(GetDeviceOrdinal(hDevice, AllocProps.location.id), + UR_RESULT_ERROR_INVALID_DEVICE); + + CUmemGenericAllocationHandle ResHandle; + UR_CHECK_ERROR(cuMemCreate(&ResHandle, size, &AllocProps, 0)); + *phPhysicalMem = new ur_physical_mem_handle_t_(ResHandle, hContext); + + return UR_RESULT_SUCCESS; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urPhysicalMemRetain(ur_physical_mem_handle_t hPhysicalMem) { + hPhysicalMem->incrementReferenceCount(); + return UR_RESULT_SUCCESS; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urPhysicalMemRelease(ur_physical_mem_handle_t hPhysicalMem) { + if (hPhysicalMem->decrementReferenceCount() > 0) + return UR_RESULT_SUCCESS; + + try { + std::unique_ptr PhysicalMemGuard(hPhysicalMem); + + ScopedContext Active(hPhysicalMem->getContext()); + UR_CHECK_ERROR(cuMemRelease(hPhysicalMem->get())); + return UR_RESULT_SUCCESS; + } catch (ur_result_t err) { + return err; + } catch (...) { + return UR_RESULT_ERROR_OUT_OF_RESOURCES; + } +} diff --git a/source/adapters/cuda/physical_mem.hpp b/source/adapters/cuda/physical_mem.hpp new file mode 100644 index 0000000000..2b0dc029d5 --- /dev/null +++ b/source/adapters/cuda/physical_mem.hpp @@ -0,0 +1,68 @@ +//===---------- physical_mem.hpp - CUDA Adapter ---------------------------===// +// +// Copyright (C) 2023 Intel Corporation +// +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See LICENSE.TXT +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#pragma once + +#include + +#include + +#include "adapter.hpp" +#include "device.hpp" +#include "platform.hpp" + +/// UR queue mapping on physical memory allocations used in virtual memory +/// management. +/// +struct ur_physical_mem_handle_t_ { + using native_type = CUmemGenericAllocationHandle; + + std::atomic_uint32_t RefCount; + native_type PhysicalMem; + ur_context_handle_t_ *Context; + + ur_physical_mem_handle_t_(native_type PhysMem, ur_context_handle_t_ *Ctx) + : RefCount(1), PhysicalMem(PhysMem), Context(Ctx) { + urContextRetain(Context); + } + + ~ur_physical_mem_handle_t_() { urContextRelease(Context); } + + native_type get() const noexcept { return PhysicalMem; } + + ur_context_handle_t_ *getContext() const noexcept { return Context; } + + uint32_t incrementReferenceCount() noexcept { return ++RefCount; } + + uint32_t decrementReferenceCount() noexcept { return --RefCount; } + + uint32_t getReferenceCount() const noexcept { return RefCount; } +}; + +// Find a device ordinal of a device. +inline ur_result_t GetDeviceOrdinal(ur_device_handle_t Device, int &Ordinal) { + ur_adapter_handle_t AdapterHandle = &adapter; + // Get list of platforms + uint32_t NumPlatforms; + UR_ASSERT(urPlatformGet(&AdapterHandle, 1, 0, nullptr, &NumPlatforms), + UR_RESULT_ERROR_INVALID_ARGUMENT); + UR_ASSERT(NumPlatforms, UR_RESULT_ERROR_UNKNOWN); + + std::vector Platforms{NumPlatforms}; + UR_ASSERT( + urPlatformGet(&AdapterHandle, 1, NumPlatforms, Platforms.data(), nullptr), + UR_RESULT_ERROR_INVALID_ARGUMENT); + + // Ordinal corresponds to the platform ID as each device has its own platform. + CUdevice NativeDevice = Device->get(); + for (Ordinal = 0; size_t(Ordinal) < Platforms.size(); ++Ordinal) + if (Platforms[Ordinal]->Devices[0]->get() == NativeDevice) + return UR_RESULT_SUCCESS; + return UR_RESULT_ERROR_INVALID_DEVICE; +} diff --git a/source/adapters/cuda/virtual_mem.cpp b/source/adapters/cuda/virtual_mem.cpp new file mode 100644 index 0000000000..583594fea1 --- /dev/null +++ b/source/adapters/cuda/virtual_mem.cpp @@ -0,0 +1,138 @@ +//===--------- virtual_mem.cpp - CUDA Adapter -----------------------------===// +// +// Copyright (C) 2023 Intel Corporation +// +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See LICENSE.TXT +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "common.hpp" +#include "context.hpp" +#include "event.hpp" +#include "physical_mem.hpp" + +#include +#include + +UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemGranularityGetInfo( + ur_context_handle_t hContext, ur_device_handle_t hDevice, + ur_virtual_mem_granularity_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet) { + UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); + + ScopedContext Active(hContext); + switch (propName) { + case UR_VIRTUAL_MEM_GRANULARITY_INFO_MINIMUM: + case UR_VIRTUAL_MEM_GRANULARITY_INFO_RECOMMENDED: { + CUmemAllocationGranularity_flags Flags = + propName == UR_VIRTUAL_MEM_GRANULARITY_INFO_MINIMUM + ? CU_MEM_ALLOC_GRANULARITY_MINIMUM + : CU_MEM_ALLOC_GRANULARITY_RECOMMENDED; + CUmemAllocationProp AllocProps = {}; + AllocProps.location.type = CU_MEM_LOCATION_TYPE_DEVICE; + AllocProps.type = CU_MEM_ALLOCATION_TYPE_PINNED; + UR_ASSERT(GetDeviceOrdinal(hDevice, AllocProps.location.id), + UR_RESULT_ERROR_INVALID_DEVICE); + + size_t Granularity; + UR_CHECK_ERROR( + cuMemGetAllocationGranularity(&Granularity, &AllocProps, Flags)); + return ReturnValue(Granularity); + } + default: + return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; + } + + return UR_RESULT_SUCCESS; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urVirtualMemReserve(ur_context_handle_t hContext, const void *pStart, + size_t size, void **ppStart) { + ScopedContext Active(hContext); + return UR_CHECK_ERROR(cuMemAddressReserve((CUdeviceptr *)ppStart, size, 0, + (CUdeviceptr)pStart, 0)); + return UR_RESULT_SUCCESS; +} + +UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemFree( + ur_context_handle_t hContext, const void *pStart, size_t size) { + ScopedContext Active(hContext); + UR_CHECK_ERROR(cuMemAddressFree((CUdeviceptr)pStart, size)); + return UR_RESULT_SUCCESS; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urVirtualMemSetAccess(ur_context_handle_t hContext, const void *pStart, + size_t size, ur_virtual_mem_access_flags_t flags) { + CUmemAccessDesc AccessDesc; + if (flags & UR_VIRTUAL_MEM_ACCESS_FLAG_READ_WRITE) + AccessDesc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE; + else if (flags & UR_VIRTUAL_MEM_ACCESS_FLAG_READ_ONLY) + AccessDesc.flags = CU_MEM_ACCESS_FLAGS_PROT_READ; + else + AccessDesc.flags = CU_MEM_ACCESS_FLAGS_PROT_NONE; + AccessDesc.location.type = CU_MEM_LOCATION_TYPE_DEVICE; + // TODO: When contexts support multiple devices, we should create a descriptor + // for each. We may also introduce a variant of this function with a + // specific device. + UR_ASSERT(GetDeviceOrdinal(hContext->getDevice(), AccessDesc.location.id), + UR_RESULT_ERROR_INVALID_DEVICE); + + ScopedContext Active(hContext); + UR_CHECK_ERROR(cuMemSetAccess((CUdeviceptr)pStart, size, &AccessDesc, 1)); + return UR_RESULT_SUCCESS; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urVirtualMemMap(ur_context_handle_t hContext, const void *pStart, size_t size, + ur_physical_mem_handle_t hPhysicalMem, size_t offset, + ur_virtual_mem_access_flags_t flags) { + ScopedContext Active(hContext); + UR_CHECK_ERROR( + cuMemMap((CUdeviceptr)pStart, size, offset, hPhysicalMem->get(), 0)); + if (flags) + UR_ASSERT(urVirtualMemSetAccess(hContext, pStart, size, flags), + UR_RESULT_ERROR_INVALID_ARGUMENT); + return UR_RESULT_SUCCESS; +} + +UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemUnmap( + ur_context_handle_t hContext, const void *pStart, size_t size) { + ScopedContext Active(hContext); + UR_CHECK_ERROR(cuMemUnmap((CUdeviceptr)pStart, size)); + return UR_RESULT_SUCCESS; +} + +UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemGetInfo( + ur_context_handle_t hContext, const void *pStart, + [[maybe_unused]] size_t size, ur_virtual_mem_info_t propName, + size_t propSize, void *pPropValue, size_t *pPropSizeRet) { + UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); + + ScopedContext Active(hContext); + switch (propName) { + case UR_VIRTUAL_MEM_INFO_ACCESS_MODE: { + CUmemLocation MemLocation = {}; + MemLocation.type = CU_MEM_LOCATION_TYPE_DEVICE; + UR_ASSERT(GetDeviceOrdinal(hContext->getDevice(), MemLocation.id), + UR_RESULT_ERROR_INVALID_DEVICE); + + unsigned long long CuAccessFlags; + UR_CHECK_ERROR( + cuMemGetAccess(&CuAccessFlags, &MemLocation, (CUdeviceptr)pStart)); + + ur_virtual_mem_access_flags_t UrAccessFlags = 0; + if (CuAccessFlags == CU_MEM_ACCESS_FLAGS_PROT_READWRITE) + UrAccessFlags = UR_VIRTUAL_MEM_ACCESS_FLAG_READ_WRITE; + else if (CuAccessFlags == CU_MEM_ACCESS_FLAGS_PROT_READ) + UrAccessFlags = UR_VIRTUAL_MEM_ACCESS_FLAG_READ_ONLY; + return ReturnValue(UrAccessFlags); + } + default: + return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; + } + return UR_RESULT_SUCCESS; +} diff --git a/source/adapters/hip/CMakeLists.txt b/source/adapters/hip/CMakeLists.txt index 7de1b5f501..1ed9d52c2b 100644 --- a/source/adapters/hip/CMakeLists.txt +++ b/source/adapters/hip/CMakeLists.txt @@ -61,6 +61,8 @@ add_ur_adapter(${TARGET_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/kernel.cpp ${CMAKE_CURRENT_SOURCE_DIR}/memory.hpp ${CMAKE_CURRENT_SOURCE_DIR}/memory.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.cpp ${CMAKE_CURRENT_SOURCE_DIR}/platform.hpp ${CMAKE_CURRENT_SOURCE_DIR}/platform.cpp ${CMAKE_CURRENT_SOURCE_DIR}/program.hpp @@ -71,6 +73,7 @@ add_ur_adapter(${TARGET_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/sampler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/usm.cpp ${CMAKE_CURRENT_SOURCE_DIR}/usm_p2p.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/virtual_mem.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../ur/ur.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../ur/ur.hpp ) diff --git a/source/adapters/hip/device.cpp b/source/adapters/hip/device.cpp index 7cec6def8b..d199e90757 100644 --- a/source/adapters/hip/device.cpp +++ b/source/adapters/hip/device.cpp @@ -809,6 +809,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_HOST_PIPE_READ_WRITE_SUPPORTED: return ReturnValue(false); + case UR_DEVICE_INFO_VIRTUAL_MEMORY_SUPPORT: + return ReturnValue(false); case UR_DEVICE_INFO_ESIMD_SUPPORT: return ReturnValue(false); diff --git a/source/adapters/hip/physical_mem.cpp b/source/adapters/hip/physical_mem.cpp new file mode 100644 index 0000000000..8939d89d33 --- /dev/null +++ b/source/adapters/hip/physical_mem.cpp @@ -0,0 +1,36 @@ +//===--------- physical_mem.cpp - HIP Adapter -----------------------------===// +// +// Copyright (C) 2023 Intel Corporation +// +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See LICENSE.TXT +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "physical_mem.hpp" +#include "common.hpp" +#include "context.hpp" +#include "event.hpp" + +UR_APIEXPORT ur_result_t UR_APICALL urPhysicalMemCreate( + ur_context_handle_t, ur_device_handle_t, size_t, + const ur_physical_mem_properties_t *, ur_physical_mem_handle_t *) { + detail::ur::die( + "Virtual memory extension is not currently implemented for HIP adapter."); + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urPhysicalMemRetain(ur_physical_mem_handle_t) { + detail::ur::die( + "Virtual memory extension is not currently implemented for HIP adapter."); + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urPhysicalMemRelease(ur_physical_mem_handle_t) { + detail::ur::die( + "Virtual memory extension is not currently implemented for HIP adapter."); + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} diff --git a/source/adapters/hip/physical_mem.hpp b/source/adapters/hip/physical_mem.hpp new file mode 100644 index 0000000000..fc50836f62 --- /dev/null +++ b/source/adapters/hip/physical_mem.hpp @@ -0,0 +1,30 @@ +//===---------- physical_mem.hpp - HIP Adapter ----------------------------===// +// +// Copyright (C) 2023 Intel Corporation +// +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See LICENSE.TXT +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#pragma once + +#include "common.hpp" +#include "device.hpp" +#include "platform.hpp" + +/// UR queue mapping on physical memory allocations used in virtual memory +/// management. +/// TODO: Implement. +/// +struct ur_physical_mem_handle_t_ { + std::atomic_uint32_t RefCount; + + ur_physical_mem_handle_t_() : RefCount(1) {} + + uint32_t incrementReferenceCount() noexcept { return ++RefCount; } + + uint32_t decrementReferenceCount() noexcept { return --RefCount; } + + uint32_t getReferenceCount() const noexcept { return RefCount; } +}; diff --git a/source/adapters/hip/virtual_mem.cpp b/source/adapters/hip/virtual_mem.cpp new file mode 100644 index 0000000000..e2c4f9faf0 --- /dev/null +++ b/source/adapters/hip/virtual_mem.cpp @@ -0,0 +1,69 @@ +//===--------- virtual_mem.cpp - HIP Adapter ------------------------------===// +// +// Copyright (C) 2023 Intel Corporation +// +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See LICENSE.TXT +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "common.hpp" +#include "context.hpp" +#include "event.hpp" +#include "physical_mem.hpp" + +UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemGranularityGetInfo( + ur_context_handle_t, ur_device_handle_t, ur_virtual_mem_granularity_info_t, + size_t, void *, size_t *) { + detail::ur::die( + "Virtual memory extension is not currently implemented for HIP adapter."); + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemReserve(ur_context_handle_t, + const void *, size_t, + void **) { + detail::ur::die( + "Virtual memory extension is not currently implemented for HIP adapter."); + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemFree(ur_context_handle_t, + const void *, size_t) { + detail::ur::die( + "Virtual memory extension is not currently implemented for HIP adapter."); + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemSetAccess( + ur_context_handle_t, const void *, size_t, ur_virtual_mem_access_flags_t) { + detail::ur::die( + "Virtual memory extension is not currently implemented for HIP adapter."); + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemMap( + ur_context_handle_t, const void *, size_t, ur_physical_mem_handle_t, size_t, + ur_virtual_mem_access_flags_t) { + detail::ur::die( + "Virtual memory extension is not currently implemented for HIP adapter."); + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemUnmap(ur_context_handle_t, + const void *, size_t) { + detail::ur::die( + "Virtual memory extension is not currently implemented for HIP adapter."); + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemGetInfo(ur_context_handle_t, + const void *, size_t, + ur_virtual_mem_info_t, + size_t, void *, + size_t *) { + detail::ur::die( + "Virtual memory extension is not currently implemented for HIP adapter."); + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} \ No newline at end of file diff --git a/source/adapters/level_zero/CMakeLists.txt b/source/adapters/level_zero/CMakeLists.txt index 223692e109..b80c5aef5d 100644 --- a/source/adapters/level_zero/CMakeLists.txt +++ b/source/adapters/level_zero/CMakeLists.txt @@ -84,6 +84,7 @@ add_ur_adapter(${TARGET_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/usm.hpp ${CMAKE_CURRENT_SOURCE_DIR}/memory.hpp ${CMAKE_CURRENT_SOURCE_DIR}/kernel.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.hpp ${CMAKE_CURRENT_SOURCE_DIR}/platform.hpp ${CMAKE_CURRENT_SOURCE_DIR}/program.hpp ${CMAKE_CURRENT_SOURCE_DIR}/queue.hpp @@ -95,8 +96,10 @@ add_ur_adapter(${TARGET_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/event.cpp ${CMAKE_CURRENT_SOURCE_DIR}/usm.cpp ${CMAKE_CURRENT_SOURCE_DIR}/usm_p2p.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/virtual_mem.cpp ${CMAKE_CURRENT_SOURCE_DIR}/memory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/kernel.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.cpp ${CMAKE_CURRENT_SOURCE_DIR}/platform.cpp ${CMAKE_CURRENT_SOURCE_DIR}/program.cpp ${CMAKE_CURRENT_SOURCE_DIR}/queue.cpp diff --git a/source/adapters/level_zero/common.cpp b/source/adapters/level_zero/common.cpp index eb0f34307c..47f0e46c91 100644 --- a/source/adapters/level_zero/common.cpp +++ b/source/adapters/level_zero/common.cpp @@ -208,6 +208,9 @@ template <> ze_structure_type_t getZeStructureType() { template <> ze_structure_type_t getZeStructureType() { return ZE_STRUCTURE_TYPE_SAMPLER_DESC; } +template <> ze_structure_type_t getZeStructureType() { + return ZE_STRUCTURE_TYPE_PHYSICAL_MEM_DESC; +} template <> ze_structure_type_t getZeStructureType() { return ZE_STRUCTURE_TYPE_DRIVER_PROPERTIES; } diff --git a/source/adapters/level_zero/device.cpp b/source/adapters/level_zero/device.cpp index 35e48931b2..fd085f1973 100644 --- a/source/adapters/level_zero/device.cpp +++ b/source/adapters/level_zero/device.cpp @@ -790,6 +790,9 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo( return ReturnValue(static_cast( 0)); //__read_write attribute currently undefinde in opencl } + case UR_DEVICE_INFO_VIRTUAL_MEMORY_SUPPORT: { + return ReturnValue(static_cast(true)); + } case UR_DEVICE_INFO_ESIMD_SUPPORT: { // ESIMD is only supported by Intel GPUs. diff --git a/source/adapters/level_zero/physical_mem.cpp b/source/adapters/level_zero/physical_mem.cpp new file mode 100644 index 0000000000..d4d9792f24 --- /dev/null +++ b/source/adapters/level_zero/physical_mem.cpp @@ -0,0 +1,54 @@ +//===---------------- physical_mem.cpp - Level Zero Adapter ---------------===// +// +// Copyright (C) 2023 Intel Corporation +// +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See LICENSE.TXT +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "physical_mem.hpp" +#include "common.hpp" +#include "context.hpp" +#include "device.hpp" +#include "ur_level_zero.hpp" + +UR_APIEXPORT ur_result_t UR_APICALL urPhysicalMemCreate( + ur_context_handle_t hContext, ur_device_handle_t hDevice, size_t size, + [[maybe_unused]] const ur_physical_mem_properties_t *pProperties, + ur_physical_mem_handle_t *phPhysicalMem) { + ZeStruct PhysicalMemDesc; + PhysicalMemDesc.flags = 0; + PhysicalMemDesc.size = size; + + ze_physical_mem_handle_t ZePhysicalMem; + ZE2UR_CALL(zePhysicalMemCreate, (hContext->ZeContext, hDevice->ZeDevice, + &PhysicalMemDesc, &ZePhysicalMem)); + try { + *phPhysicalMem = new ur_physical_mem_handle_t_(ZePhysicalMem, hContext); + } catch (const std::bad_alloc &) { + return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } catch (...) { + return UR_RESULT_ERROR_UNKNOWN; + } + return UR_RESULT_SUCCESS; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urPhysicalMemRetain(ur_physical_mem_handle_t hPhysicalMem) { + hPhysicalMem->RefCount.increment(); + return UR_RESULT_SUCCESS; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urPhysicalMemRelease(ur_physical_mem_handle_t hPhysicalMem) { + if (!hPhysicalMem->RefCount.decrementAndTest()) + return UR_RESULT_SUCCESS; + + ZE2UR_CALL(zePhysicalMemDestroy, + (hPhysicalMem->Context->ZeContext, hPhysicalMem->ZePhysicalMem)); + delete hPhysicalMem; + + return UR_RESULT_SUCCESS; +} diff --git a/source/adapters/level_zero/physical_mem.hpp b/source/adapters/level_zero/physical_mem.hpp new file mode 100644 index 0000000000..9b83d93222 --- /dev/null +++ b/source/adapters/level_zero/physical_mem.hpp @@ -0,0 +1,24 @@ +//===---------------- physical_mem.hpp - Level Zero Adapter ---------------===// +// +// Copyright (C) 2023 Intel Corporation +// +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See LICENSE.TXT +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#pragma once + +#include "common.hpp" + +struct ur_physical_mem_handle_t_ : _ur_object { + ur_physical_mem_handle_t_(ze_physical_mem_handle_t ZePhysicalMem, + ur_context_handle_t Context) + : ZePhysicalMem{ZePhysicalMem}, Context{Context} {} + + // Level Zero physical memory handle. + ze_physical_mem_handle_t ZePhysicalMem; + + // Keeps the PI context of this memory handle. + ur_context_handle_t Context; +}; diff --git a/source/adapters/level_zero/ur_level_zero.hpp b/source/adapters/level_zero/ur_level_zero.hpp index 38bc5aed2d..dd7bbf67b3 100644 --- a/source/adapters/level_zero/ur_level_zero.hpp +++ b/source/adapters/level_zero/ur_level_zero.hpp @@ -30,6 +30,7 @@ #include "image.hpp" #include "kernel.hpp" #include "memory.hpp" +#include "physical_mem.hpp" #include "platform.hpp" #include "program.hpp" #include "queue.hpp" diff --git a/source/adapters/level_zero/virtual_mem.cpp b/source/adapters/level_zero/virtual_mem.cpp new file mode 100644 index 0000000000..e90aec45de --- /dev/null +++ b/source/adapters/level_zero/virtual_mem.cpp @@ -0,0 +1,120 @@ +//===---------------- virtual_mem.cpp - Level Zero Adapter ----------------===// +// +// Copyright (C) 2023 Intel Corporation +// +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See LICENSE.TXT +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "common.hpp" +#include "context.hpp" +#include "device.hpp" +#include "physical_mem.hpp" +#include "ur_level_zero.hpp" + +UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemGranularityGetInfo( + ur_context_handle_t hContext, ur_device_handle_t hDevice, + ur_virtual_mem_granularity_info_t propName, size_t propSize, + void *pPropValue, size_t *pPropSizeRet) { + UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); + switch (propName) { + case UR_VIRTUAL_MEM_GRANULARITY_INFO_MINIMUM: + case UR_VIRTUAL_MEM_GRANULARITY_INFO_RECOMMENDED: { + // For L0 the minimum and recommended granularity is the same. We use an + // memory size of 1 byte to get the actual granularity instead of the + // aligned size. + size_t PageSize; + ZE2UR_CALL(zeVirtualMemQueryPageSize, + (hContext->ZeContext, hDevice->ZeDevice, 1, &PageSize)); + return ReturnValue(PageSize); + } + default: + urPrint("Unsupported propName in urQueueGetInfo: propName=%d(0x%x)\n", + propName, propName); + return UR_RESULT_ERROR_INVALID_VALUE; + } + return UR_RESULT_SUCCESS; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urVirtualMemReserve(ur_context_handle_t hContext, const void *pStart, + size_t size, void **ppStart) { + ZE2UR_CALL(zeVirtualMemReserve, (hContext->ZeContext, pStart, size, ppStart)); + + return UR_RESULT_SUCCESS; +} + +UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemFree( + ur_context_handle_t hContext, const void *pStart, size_t size) { + ZE2UR_CALL(zeVirtualMemFree, (hContext->ZeContext, pStart, size)); + + return UR_RESULT_SUCCESS; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urVirtualMemSetAccess(ur_context_handle_t hContext, const void *pStart, + size_t size, ur_virtual_mem_access_flags_t flags) { + ze_memory_access_attribute_t AccessAttr = ZE_MEMORY_ACCESS_ATTRIBUTE_NONE; + if (flags & UR_VIRTUAL_MEM_ACCESS_FLAG_READ_WRITE) + AccessAttr = ZE_MEMORY_ACCESS_ATTRIBUTE_READWRITE; + else if (flags & UR_VIRTUAL_MEM_ACCESS_FLAG_READ_ONLY) + AccessAttr = ZE_MEMORY_ACCESS_ATTRIBUTE_READONLY; + + ZE2UR_CALL(zeVirtualMemSetAccessAttribute, + (hContext->ZeContext, pStart, size, AccessAttr)); + + return UR_RESULT_SUCCESS; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urVirtualMemMap(ur_context_handle_t hContext, const void *pStart, size_t size, + ur_physical_mem_handle_t hPhysicalMem, size_t offset, + ur_virtual_mem_access_flags_t flags) { + ze_memory_access_attribute_t AccessAttr = ZE_MEMORY_ACCESS_ATTRIBUTE_NONE; + if (flags & UR_VIRTUAL_MEM_ACCESS_FLAG_READ_WRITE) + AccessAttr = ZE_MEMORY_ACCESS_ATTRIBUTE_READWRITE; + else if (flags & UR_VIRTUAL_MEM_ACCESS_FLAG_READ_ONLY) + AccessAttr = ZE_MEMORY_ACCESS_ATTRIBUTE_READONLY; + + ZE2UR_CALL(zeVirtualMemMap, + (hContext->ZeContext, pStart, size, hPhysicalMem->ZePhysicalMem, + offset, AccessAttr)); + + return UR_RESULT_SUCCESS; +} + +UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemUnmap( + ur_context_handle_t hContext, const void *pStart, size_t size) { + ZE2UR_CALL(zeVirtualMemUnmap, (hContext->ZeContext, pStart, size)); + + return UR_RESULT_SUCCESS; +} + +UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemGetInfo( + ur_context_handle_t hContext, const void *pStart, + [[maybe_unused]] size_t size, ur_virtual_mem_info_t propName, + size_t propSize, void *pPropValue, size_t *pPropSizeRet) { + UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); + switch (propName) { + case UR_VIRTUAL_MEM_INFO_ACCESS_MODE: { + size_t QuerySize; + ze_memory_access_attribute_t Access; + ZE2UR_CALL(zeVirtualMemGetAccessAttribute, + (hContext->ZeContext, pStart, size, &Access, &QuerySize)); + ur_virtual_mem_access_flags_t RetFlags = 0; + if (Access & ZE_MEMORY_ACCESS_ATTRIBUTE_READWRITE) + RetFlags |= UR_VIRTUAL_MEM_ACCESS_FLAG_READ_WRITE; + if (Access & ZE_MEMORY_ACCESS_ATTRIBUTE_READONLY) + RetFlags |= UR_VIRTUAL_MEM_ACCESS_FLAG_READ_ONLY; + return ReturnValue(Access); + } + default: + urPrint("Unsupported propName in urQueueGetInfo: propName=%d(0x%x)\n", + propName, propName); + return UR_RESULT_ERROR_INVALID_VALUE; + } + + return UR_RESULT_SUCCESS; +} From 1096b0f389e661b12a5c30c3edc5f84f09c9d32c Mon Sep 17 00:00:00 2001 From: "Larsen, Steffen" Date: Mon, 9 Oct 2023 10:09:34 -0700 Subject: [PATCH 04/72] Remove duplicate CUDA UR_DEVICE_INFO_VIRTUAL_MEMORY_SUPPORT case Signed-off-by: Larsen, Steffen --- source/adapters/cuda/device.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/source/adapters/cuda/device.cpp b/source/adapters/cuda/device.cpp index c27dbdbb52..9677005d07 100644 --- a/source/adapters/cuda/device.cpp +++ b/source/adapters/cuda/device.cpp @@ -1028,7 +1028,6 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, case UR_DEVICE_INFO_GPU_SUBSLICES_PER_SLICE: case UR_DEVICE_INFO_GPU_EU_COUNT_PER_SUBSLICE: case UR_DEVICE_INFO_GPU_HW_THREADS_PER_EU: - case UR_DEVICE_INFO_VIRTUAL_MEMORY_SUPPORT: return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; default: From c36bd72e3c5e94f0bc4756789d676a4d20b1dc43 Mon Sep 17 00:00:00 2001 From: "Larsen, Steffen" Date: Mon, 9 Oct 2023 10:14:47 -0700 Subject: [PATCH 05/72] Fix return of check error Signed-off-by: Larsen, Steffen --- source/adapters/cuda/virtual_mem.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/adapters/cuda/virtual_mem.cpp b/source/adapters/cuda/virtual_mem.cpp index 583594fea1..203d052619 100644 --- a/source/adapters/cuda/virtual_mem.cpp +++ b/source/adapters/cuda/virtual_mem.cpp @@ -52,8 +52,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemReserve(ur_context_handle_t hContext, const void *pStart, size_t size, void **ppStart) { ScopedContext Active(hContext); - return UR_CHECK_ERROR(cuMemAddressReserve((CUdeviceptr *)ppStart, size, 0, - (CUdeviceptr)pStart, 0)); + UR_CHECK_ERROR(cuMemAddressReserve((CUdeviceptr *)ppStart, size, 0, + (CUdeviceptr)pStart, 0)); return UR_RESULT_SUCCESS; } From 6ba94ac426e3fb258f423be87e4a771f6db70060 Mon Sep 17 00:00:00 2001 From: "Larsen, Steffen" Date: Tue, 10 Oct 2023 03:38:20 -0700 Subject: [PATCH 06/72] Fix common include in cuda device.hpp Signed-off-by: Larsen, Steffen --- source/adapters/cuda/device.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/adapters/cuda/device.hpp b/source/adapters/cuda/device.hpp index 5ad70672e9..08a1b5852a 100644 --- a/source/adapters/cuda/device.hpp +++ b/source/adapters/cuda/device.hpp @@ -11,6 +11,8 @@ #include +#include "common.hpp" + struct ur_device_handle_t_ { private: using native_type = CUdevice; From 9ae8a992a9b40e04ba6b98b672b6f3e7e385e849 Mon Sep 17 00:00:00 2001 From: "Larsen, Steffen" Date: Wed, 11 Oct 2023 08:25:24 -0700 Subject: [PATCH 07/72] Zero-initialize CUmemAccessDesc Signed-off-by: Larsen, Steffen --- source/adapters/cuda/virtual_mem.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/adapters/cuda/virtual_mem.cpp b/source/adapters/cuda/virtual_mem.cpp index 203d052619..f49da3132e 100644 --- a/source/adapters/cuda/virtual_mem.cpp +++ b/source/adapters/cuda/virtual_mem.cpp @@ -67,7 +67,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemFree( UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemSetAccess(ur_context_handle_t hContext, const void *pStart, size_t size, ur_virtual_mem_access_flags_t flags) { - CUmemAccessDesc AccessDesc; + CUmemAccessDesc AccessDesc = {}; if (flags & UR_VIRTUAL_MEM_ACCESS_FLAG_READ_WRITE) AccessDesc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE; else if (flags & UR_VIRTUAL_MEM_ACCESS_FLAG_READ_ONLY) From 9f90e435e43daf74ebd04ff48daa544a4afbaa80 Mon Sep 17 00:00:00 2001 From: "Larsen, Steffen" Date: Wed, 11 Oct 2023 08:26:12 -0700 Subject: [PATCH 08/72] Add missing newline Signed-off-by: Larsen, Steffen --- source/adapters/hip/virtual_mem.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/adapters/hip/virtual_mem.cpp b/source/adapters/hip/virtual_mem.cpp index e2c4f9faf0..6330451797 100644 --- a/source/adapters/hip/virtual_mem.cpp +++ b/source/adapters/hip/virtual_mem.cpp @@ -66,4 +66,4 @@ UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemGetInfo(ur_context_handle_t, detail::ur::die( "Virtual memory extension is not currently implemented for HIP adapter."); return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} \ No newline at end of file +} From 336ce895dcf3f0f2f9a96e81ac372f86f92cbc86 Mon Sep 17 00:00:00 2001 From: "Larsen, Steffen" Date: Fri, 13 Oct 2023 02:44:01 -0700 Subject: [PATCH 09/72] Fix wrongful use of UR_ASSERT in cuda implementation Signed-off-by: Larsen, Steffen --- source/adapters/cuda/physical_mem.cpp | 3 +-- source/adapters/cuda/physical_mem.hpp | 8 +++----- source/adapters/cuda/virtual_mem.cpp | 13 +++++-------- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/source/adapters/cuda/physical_mem.cpp b/source/adapters/cuda/physical_mem.cpp index 177bc2234d..444d492aa3 100644 --- a/source/adapters/cuda/physical_mem.cpp +++ b/source/adapters/cuda/physical_mem.cpp @@ -23,8 +23,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urPhysicalMemCreate( CUmemAllocationProp AllocProps = {}; AllocProps.location.type = CU_MEM_LOCATION_TYPE_DEVICE; AllocProps.type = CU_MEM_ALLOCATION_TYPE_PINNED; - UR_ASSERT(GetDeviceOrdinal(hDevice, AllocProps.location.id), - UR_RESULT_ERROR_INVALID_DEVICE); + UR_CHECK_ERROR(GetDeviceOrdinal(hDevice, AllocProps.location.id)); CUmemGenericAllocationHandle ResHandle; UR_CHECK_ERROR(cuMemCreate(&ResHandle, size, &AllocProps, 0)); diff --git a/source/adapters/cuda/physical_mem.hpp b/source/adapters/cuda/physical_mem.hpp index 2b0dc029d5..0ce332e112 100644 --- a/source/adapters/cuda/physical_mem.hpp +++ b/source/adapters/cuda/physical_mem.hpp @@ -50,14 +50,12 @@ inline ur_result_t GetDeviceOrdinal(ur_device_handle_t Device, int &Ordinal) { ur_adapter_handle_t AdapterHandle = &adapter; // Get list of platforms uint32_t NumPlatforms; - UR_ASSERT(urPlatformGet(&AdapterHandle, 1, 0, nullptr, &NumPlatforms), - UR_RESULT_ERROR_INVALID_ARGUMENT); + UR_CHECK_ERROR(urPlatformGet(&AdapterHandle, 1, 0, nullptr, &NumPlatforms)); UR_ASSERT(NumPlatforms, UR_RESULT_ERROR_UNKNOWN); std::vector Platforms{NumPlatforms}; - UR_ASSERT( - urPlatformGet(&AdapterHandle, 1, NumPlatforms, Platforms.data(), nullptr), - UR_RESULT_ERROR_INVALID_ARGUMENT); + UR_CHECK_ERROR(urPlatformGet(&AdapterHandle, 1, NumPlatforms, + Platforms.data(), nullptr)); // Ordinal corresponds to the platform ID as each device has its own platform. CUdevice NativeDevice = Device->get(); diff --git a/source/adapters/cuda/virtual_mem.cpp b/source/adapters/cuda/virtual_mem.cpp index f49da3132e..9c37dda4fb 100644 --- a/source/adapters/cuda/virtual_mem.cpp +++ b/source/adapters/cuda/virtual_mem.cpp @@ -33,8 +33,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemGranularityGetInfo( CUmemAllocationProp AllocProps = {}; AllocProps.location.type = CU_MEM_LOCATION_TYPE_DEVICE; AllocProps.type = CU_MEM_ALLOCATION_TYPE_PINNED; - UR_ASSERT(GetDeviceOrdinal(hDevice, AllocProps.location.id), - UR_RESULT_ERROR_INVALID_DEVICE); + UR_CHECK_ERROR(GetDeviceOrdinal(hDevice, AllocProps.location.id)); size_t Granularity; UR_CHECK_ERROR( @@ -78,8 +77,8 @@ urVirtualMemSetAccess(ur_context_handle_t hContext, const void *pStart, // TODO: When contexts support multiple devices, we should create a descriptor // for each. We may also introduce a variant of this function with a // specific device. - UR_ASSERT(GetDeviceOrdinal(hContext->getDevice(), AccessDesc.location.id), - UR_RESULT_ERROR_INVALID_DEVICE); + UR_CHECK_ERROR( + GetDeviceOrdinal(hContext->getDevice(), AccessDesc.location.id)); ScopedContext Active(hContext); UR_CHECK_ERROR(cuMemSetAccess((CUdeviceptr)pStart, size, &AccessDesc, 1)); @@ -94,8 +93,7 @@ urVirtualMemMap(ur_context_handle_t hContext, const void *pStart, size_t size, UR_CHECK_ERROR( cuMemMap((CUdeviceptr)pStart, size, offset, hPhysicalMem->get(), 0)); if (flags) - UR_ASSERT(urVirtualMemSetAccess(hContext, pStart, size, flags), - UR_RESULT_ERROR_INVALID_ARGUMENT); + UR_CHECK_ERROR(urVirtualMemSetAccess(hContext, pStart, size, flags)); return UR_RESULT_SUCCESS; } @@ -117,8 +115,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemGetInfo( case UR_VIRTUAL_MEM_INFO_ACCESS_MODE: { CUmemLocation MemLocation = {}; MemLocation.type = CU_MEM_LOCATION_TYPE_DEVICE; - UR_ASSERT(GetDeviceOrdinal(hContext->getDevice(), MemLocation.id), - UR_RESULT_ERROR_INVALID_DEVICE); + UR_CHECK_ERROR(GetDeviceOrdinal(hContext->getDevice(), MemLocation.id)); unsigned long long CuAccessFlags; UR_CHECK_ERROR( From 84a3afa631728a72470d42fef795aabecb6a7dd4 Mon Sep 17 00:00:00 2001 From: "Larsen, Steffen" Date: Fri, 13 Oct 2023 02:45:00 -0700 Subject: [PATCH 10/72] an -> a Signed-off-by: Larsen, Steffen --- source/adapters/level_zero/virtual_mem.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/adapters/level_zero/virtual_mem.cpp b/source/adapters/level_zero/virtual_mem.cpp index e90aec45de..545f9fde54 100644 --- a/source/adapters/level_zero/virtual_mem.cpp +++ b/source/adapters/level_zero/virtual_mem.cpp @@ -22,7 +22,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemGranularityGetInfo( switch (propName) { case UR_VIRTUAL_MEM_GRANULARITY_INFO_MINIMUM: case UR_VIRTUAL_MEM_GRANULARITY_INFO_RECOMMENDED: { - // For L0 the minimum and recommended granularity is the same. We use an + // For L0 the minimum and recommended granularity is the same. We use a // memory size of 1 byte to get the actual granularity instead of the // aligned size. size_t PageSize; From a8dadc15a97d39777a8f27dce9c9b6a9ce5b307b Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Wed, 1 Nov 2023 23:23:37 -0700 Subject: [PATCH 11/72] UR Sanitizer: fix bug --- .../level_zero/ur_interface_loader.cpp | 8 +- .../sanitizer/sanitizer_interceptor.cpp | 218 +++++++++++++++--- 2 files changed, 191 insertions(+), 35 deletions(-) diff --git a/source/adapters/level_zero/ur_interface_loader.cpp b/source/adapters/level_zero/ur_interface_loader.cpp index 5f6da8fd86..439deb055c 100644 --- a/source/adapters/level_zero/ur_interface_loader.cpp +++ b/source/adapters/level_zero/ur_interface_loader.cpp @@ -405,9 +405,9 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetVirtualMemProcAddrTable( pDdiTable->pfnFree = nullptr; pDdiTable->pfnGetInfo = nullptr; - pDdiTable->pfnGranularityGetInfo = nullptr; - pDdiTable->pfnMap = nullptr; - pDdiTable->pfnReserve = nullptr; + pDdiTable->pfnGranularityGetInfo = urVirtualMemGranularityGetInfo; + pDdiTable->pfnMap = urVirtualMemMap; + pDdiTable->pfnReserve = urVirtualMemReserve; pDdiTable->pfnSetAccess = nullptr; pDdiTable->pfnUnmap = nullptr; @@ -424,7 +424,7 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetPhysicalMemProcAddrTable( return retVal; } - pDdiTable->pfnCreate = nullptr; + pDdiTable->pfnCreate = urPhysicalMemCreate; pDdiTable->pfnRelease = nullptr; pDdiTable->pfnRetain = nullptr; diff --git a/source/loader/layers/sanitizer/sanitizer_interceptor.cpp b/source/loader/layers/sanitizer/sanitizer_interceptor.cpp index 29454b34a2..490998fce5 100644 --- a/source/loader/layers/sanitizer/sanitizer_interceptor.cpp +++ b/source/loader/layers/sanitizer/sanitizer_interceptor.cpp @@ -140,6 +140,149 @@ inline constexpr uptr MemToShadow(uptr Addr, uptr ShadowOffset) { return ShadowOffset + ((Addr) >> ASAN_SHADOW_SCALE); } +static auto getUrResultString = [](ur_result_t Result) { + switch (Result) { + case UR_RESULT_SUCCESS: + return "UR_RESULT_SUCCESS"; + case UR_RESULT_ERROR_INVALID_OPERATION: + return "UR_RESULT_ERROR_INVALID_OPERATION"; + case UR_RESULT_ERROR_INVALID_QUEUE_PROPERTIES: + return "UR_RESULT_ERROR_INVALID_QUEUE_PROPERTIES"; + case UR_RESULT_ERROR_INVALID_QUEUE: + return "UR_RESULT_ERROR_INVALID_QUEUE"; + case UR_RESULT_ERROR_INVALID_VALUE: + return "UR_RESULT_ERROR_INVALID_VALUE"; + case UR_RESULT_ERROR_INVALID_CONTEXT: + return "UR_RESULT_ERROR_INVALID_CONTEXT"; + case UR_RESULT_ERROR_INVALID_PLATFORM: + return "UR_RESULT_ERROR_INVALID_PLATFORM"; + case UR_RESULT_ERROR_INVALID_BINARY: + return "UR_RESULT_ERROR_INVALID_BINARY"; + case UR_RESULT_ERROR_INVALID_PROGRAM: + return "UR_RESULT_ERROR_INVALID_PROGRAM"; + case UR_RESULT_ERROR_INVALID_SAMPLER: + return "UR_RESULT_ERROR_INVALID_SAMPLER"; + case UR_RESULT_ERROR_INVALID_BUFFER_SIZE: + return "UR_RESULT_ERROR_INVALID_BUFFER_SIZE"; + case UR_RESULT_ERROR_INVALID_MEM_OBJECT: + return "UR_RESULT_ERROR_INVALID_MEM_OBJECT"; + case UR_RESULT_ERROR_INVALID_EVENT: + return "UR_RESULT_ERROR_INVALID_EVENT"; + case UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST: + return "UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST"; + case UR_RESULT_ERROR_MISALIGNED_SUB_BUFFER_OFFSET: + return "UR_RESULT_ERROR_MISALIGNED_SUB_BUFFER_OFFSET"; + case UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE: + return "UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE"; + case UR_RESULT_ERROR_COMPILER_NOT_AVAILABLE: + return "UR_RESULT_ERROR_COMPILER_NOT_AVAILABLE"; + case UR_RESULT_ERROR_PROFILING_INFO_NOT_AVAILABLE: + return "UR_RESULT_ERROR_PROFILING_INFO_NOT_AVAILABLE"; + case UR_RESULT_ERROR_DEVICE_NOT_FOUND: + return "UR_RESULT_ERROR_DEVICE_NOT_FOUND"; + case UR_RESULT_ERROR_INVALID_DEVICE: + return "UR_RESULT_ERROR_INVALID_DEVICE"; + case UR_RESULT_ERROR_DEVICE_LOST: + return "UR_RESULT_ERROR_DEVICE_LOST"; + case UR_RESULT_ERROR_DEVICE_REQUIRES_RESET: + return "UR_RESULT_ERROR_DEVICE_REQUIRES_RESET"; + case UR_RESULT_ERROR_DEVICE_IN_LOW_POWER_STATE: + return "UR_RESULT_ERROR_DEVICE_IN_LOW_POWER_STATE"; + case UR_RESULT_ERROR_DEVICE_PARTITION_FAILED: + return "UR_RESULT_ERROR_DEVICE_PARTITION_FAILED"; + case UR_RESULT_ERROR_INVALID_DEVICE_PARTITION_COUNT: + return "UR_RESULT_ERROR_INVALID_DEVICE_PARTITION_COUNT"; + case UR_RESULT_ERROR_INVALID_WORK_ITEM_SIZE: + return "UR_RESULT_ERROR_INVALID_WORK_ITEM_SIZE"; + case UR_RESULT_ERROR_INVALID_WORK_DIMENSION: + return "UR_RESULT_ERROR_INVALID_WORK_DIMENSION"; + case UR_RESULT_ERROR_INVALID_KERNEL_ARGS: + return "UR_RESULT_ERROR_INVALID_KERNEL_ARGS"; + case UR_RESULT_ERROR_INVALID_KERNEL: + return "UR_RESULT_ERROR_INVALID_KERNEL"; + case UR_RESULT_ERROR_INVALID_KERNEL_NAME: + return "UR_RESULT_ERROR_INVALID_KERNEL_NAME"; + case UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX: + return "UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX"; + case UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_SIZE: + return "UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_SIZE"; + case UR_RESULT_ERROR_INVALID_KERNEL_ATTRIBUTE_VALUE: + return "UR_RESULT_ERROR_INVALID_KERNEL_ATTRIBUTE_VALUE"; + case UR_RESULT_ERROR_INVALID_IMAGE_SIZE: + return "UR_RESULT_ERROR_INVALID_IMAGE_SIZE"; + case UR_RESULT_ERROR_INVALID_IMAGE_FORMAT_DESCRIPTOR: + return "UR_RESULT_ERROR_INVALID_IMAGE_FORMAT_DESCRIPTOR"; + case UR_RESULT_ERROR_IMAGE_FORMAT_NOT_SUPPORTED: + return "UR_RESULT_ERROR_IMAGE_FORMAT_NOT_SUPPORTED"; + case UR_RESULT_ERROR_MEM_OBJECT_ALLOCATION_FAILURE: + return "UR_RESULT_ERROR_MEM_OBJECT_ALLOCATION_FAILURE"; + case UR_RESULT_ERROR_INVALID_PROGRAM_EXECUTABLE: + return "UR_RESULT_ERROR_INVALID_PROGRAM_EXECUTABLE"; + case UR_RESULT_ERROR_UNINITIALIZED: + return "UR_RESULT_ERROR_UNINITIALIZED"; + case UR_RESULT_ERROR_OUT_OF_HOST_MEMORY: + return "UR_RESULT_ERROR_OUT_OF_HOST_MEMORY"; + case UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY: + return "UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY"; + case UR_RESULT_ERROR_OUT_OF_RESOURCES: + return "UR_RESULT_ERROR_OUT_OF_RESOURCES"; + case UR_RESULT_ERROR_PROGRAM_BUILD_FAILURE: + return "UR_RESULT_ERROR_PROGRAM_BUILD_FAILURE"; + case UR_RESULT_ERROR_PROGRAM_LINK_FAILURE: + return "UR_RESULT_ERROR_PROGRAM_LINK_FAILURE"; + case UR_RESULT_ERROR_UNSUPPORTED_VERSION: + return "UR_RESULT_ERROR_UNSUPPORTED_VERSION"; + case UR_RESULT_ERROR_UNSUPPORTED_FEATURE: + return "UR_RESULT_ERROR_UNSUPPORTED_FEATURE"; + case UR_RESULT_ERROR_INVALID_ARGUMENT: + return "UR_RESULT_ERROR_INVALID_ARGUMENT"; + case UR_RESULT_ERROR_INVALID_NULL_HANDLE: + return "UR_RESULT_ERROR_INVALID_NULL_HANDLE"; + case UR_RESULT_ERROR_HANDLE_OBJECT_IN_USE: + return "UR_RESULT_ERROR_HANDLE_OBJECT_IN_USE"; + case UR_RESULT_ERROR_INVALID_NULL_POINTER: + return "UR_RESULT_ERROR_INVALID_NULL_POINTER"; + case UR_RESULT_ERROR_INVALID_SIZE: + return "UR_RESULT_ERROR_INVALID_SIZE"; + case UR_RESULT_ERROR_UNSUPPORTED_SIZE: + return "UR_RESULT_ERROR_UNSUPPORTED_SIZE"; + case UR_RESULT_ERROR_UNSUPPORTED_ALIGNMENT: + return "UR_RESULT_ERROR_UNSUPPORTED_ALIGNMENT"; + case UR_RESULT_ERROR_INVALID_SYNCHRONIZATION_OBJECT: + return "UR_RESULT_ERROR_INVALID_SYNCHRONIZATION_OBJECT"; + case UR_RESULT_ERROR_INVALID_ENUMERATION: + return "UR_RESULT_ERROR_INVALID_ENUMERATION"; + case UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION: + return "UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION"; + case UR_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT: + return "UR_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT"; + case UR_RESULT_ERROR_INVALID_NATIVE_BINARY: + return "UR_RESULT_ERROR_INVALID_NATIVE_BINARY"; + case UR_RESULT_ERROR_INVALID_GLOBAL_NAME: + return "UR_RESULT_ERROR_INVALID_GLOBAL_NAME"; + case UR_RESULT_ERROR_INVALID_FUNCTION_NAME: + return "UR_RESULT_ERROR_INVALID_FUNCTION_NAME"; + case UR_RESULT_ERROR_INVALID_GROUP_SIZE_DIMENSION: + return "UR_RESULT_ERROR_INVALID_GROUP_SIZE_DIMENSION"; + case UR_RESULT_ERROR_INVALID_GLOBAL_WIDTH_DIMENSION: + return "UR_RESULT_ERROR_INVALID_GLOBAL_WIDTH_DIMENSION"; + case UR_RESULT_ERROR_PROGRAM_UNLINKED: + return "UR_RESULT_ERROR_PROGRAM_UNLINKED"; + case UR_RESULT_ERROR_OVERLAPPING_REGIONS: + return "UR_RESULT_ERROR_OVERLAPPING_REGIONS"; + case UR_RESULT_ERROR_INVALID_HOST_PTR: + return "UR_RESULT_ERROR_INVALID_HOST_PTR"; + case UR_RESULT_ERROR_INVALID_USM_SIZE: + return "UR_RESULT_ERROR_INVALID_USM_SIZE"; + case UR_RESULT_ERROR_OBJECT_ALLOCATION_FAILURE: + return "UR_RESULT_ERROR_OBJECT_ALLOCATION_FAILURE"; + case UR_RESULT_ERROR_ADAPTER_SPECIFIC: + return "UR_RESULT_ERROR_ADAPTER_SPECIFIC"; + default: + return "UR_RESULT_ERROR_UNKNOWN"; + } +}; + } // namespace ur_result_t SanitizerInterceptor::allocateMemory( @@ -328,34 +471,44 @@ void SanitizerInterceptor::postLaunchKernel(ur_kernel_handle_t Kernel, ur_queue_handle_t Queue, ur_event_handle_t *Event, bool SetCallback) { - // auto &KernelInfo = getKernelInfo(Kernel); - // auto Program = KernelInfo.Program; + auto &KernelInfo = getKernelInfo(Kernel); + auto Program = KernelInfo.Program; - // ur_event_handle_t ReadEvent{}; + ur_event_handle_t ReadEvent{}; // If kernel has defined SPIR_DeviceSanitizerReportMem, then we try to read it // to host, but it's okay that it isn't defined - // auto Ret = m_Dditable.Enqueue.pfnDeviceGlobalVariableRead( - // Queue, Program, kSPIR_DeviceSanitizerReportMem, false, - // sizeof(SPIR_DeviceSanitizerReportMem), 0, - // &SPIR_DeviceSanitizerReportMem, 1, Event, &ReadEvent); - - // if (Ret == UR_RESULT_SUCCESS && SetCallback) { - // Ret = piEventSetCallback( - // ReadEvent, ur_event_handle_t_COMPLETE, - // [](ur_event_handle_t Event, pi_int32 EventCommandStatus, - // void *user_data) { - // (void)Event; - // (void)EventCommandStatus; - // auto *KernelName = (const char *)user_data; - // checkSanitizerReport(KernelName); - // }, - // (void *)KernelInfo.Name.c_str()); - // assert(Ret == UR_RESULT_SUCCESS); - // } - - // *Event = ReadEvent; - return; + auto Ret = m_Dditable.Enqueue.pfnDeviceGlobalVariableRead( + Queue, Program, kSPIR_DeviceSanitizerReportMem, true, + sizeof(SPIR_DeviceSanitizerReportMem), 0, + &SPIR_DeviceSanitizerReportMem, 1, Event, &ReadEvent); + + if (Ret == UR_RESULT_SUCCESS) { + *Event = ReadEvent; + + auto AH = &SPIR_DeviceSanitizerReportMem; + if (!AH->Flag) { + return; + } + + const char *File = AH->File[0] ? AH->File : ""; + const char *Func = AH->Func[0] ? AH->Func : ""; + + fprintf(stderr, "\n====ERROR: DeviceSanitizer: %s on %s\n\n", + DeviceSanitizerFormat(AH->ErrorType), + DeviceSanitizerFormat(AH->MemoryType)); + fprintf(stderr, + "%s of size %u at kernel <%s> LID(%lu, %lu, %lu) GID(%lu, " + "%lu, %lu)\n", + AH->IsWrite ? "WRITE" : "READ", AH->AccessSize, + KernelInfo.Name.c_str(), AH->LID0, AH->LID1, AH->LID2, AH->GID0, + AH->GID1, AH->GID2); + fprintf(stderr, " #0 %s %s:%d\n", Func, File, AH->Line); + fflush(stderr); + if (!AH->IsRecover) { + abort(); + } + } } std::string SanitizerInterceptor::getKernelName(ur_kernel_handle_t Kernel) { @@ -443,7 +596,7 @@ SanitizerInterceptor::piextMemAllocShadow(ur_context_handle_t Context, auto URes = m_Dditable.VirtualMem.pfnReserve( Context, nullptr, SHADOW_SIZE, (void **)&DeviceInfo.ShadowOffset); if (URes != UR_RESULT_SUCCESS) { - printf("urVirtualMemReserve(): %p\n", (void *)URes); + printf("urVirtualMemReserve(): %s\n", getUrResultString(URes)); } assert(URes == UR_RESULT_SUCCESS); @@ -491,20 +644,22 @@ ur_result_t SanitizerInterceptor::piextEnqueueMemSetShadow( auto URes = m_Dditable.PhysicalMem.pfnCreate( Context, Device, PageSize, &Desc, &PhysicalMem); if (URes != UR_RESULT_SUCCESS) { - printf(" zePhysicalMemCreate(): %p\n", (void *)URes); + printf(" zePhysicalMemCreate(): %s\n", + getUrResultString(URes)); } assert(URes == UR_RESULT_SUCCESS); } - printf(" zeVirtualMemMap: %p ~ %p\n", MappedPtr, - MappedPtr + PageSize - 1); + printf(" zeVirtualMemMap: %p ~ %p\n", (void *)MappedPtr, + (void *)(MappedPtr + PageSize - 1)); // FIXME: No flag to check the failed reason is VA is already mapped auto URes = m_Dditable.VirtualMem.pfnMap( Context, (void *)MappedPtr, PageSize, PhysicalMem, 0, UR_VIRTUAL_MEM_ACCESS_FLAG_READ_WRITE); if (URes != UR_RESULT_SUCCESS) { - printf(" zeVirtualMemMap(): %p\n", (void *)URes); + printf(" zeVirtualMemMap(): %s\n", + getUrResultString(URes)); } // Initialize to zero @@ -519,7 +674,8 @@ ur_result_t SanitizerInterceptor::piextEnqueueMemSetShadow( Queue, (void *)MappedPtr, 1, Pattern, PageSize, NumEventsInWaitList, EventsWaitList, Event); if (URes != UR_RESULT_SUCCESS) { - printf(" urEnqueueUSMFill(): %p\n", (void *)URes); + printf(" urEnqueueUSMFill(): %s\n", + getUrResultString(URes)); } assert(URes == UR_RESULT_SUCCESS); @@ -535,7 +691,7 @@ ur_result_t SanitizerInterceptor::piextEnqueueMemSetShadow( (ShadowEnd - ShadowBegin + 1), NumEventsInWaitList, EventsWaitList, Event); if (URes != UR_RESULT_SUCCESS) { - printf(" urEnqueueUSMFill(): %p\n", (void *)URes); + printf(" urEnqueueUSMFill(): %s\n", getUrResultString(URes)); } assert(URes == UR_RESULT_SUCCESS); } else { From 4d63350120b73c4f2d50a43901cc66377cb033ee Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Fri, 3 Nov 2023 05:52:13 -0700 Subject: [PATCH 12/72] UR Sanitizer: Add membuffer --- source/loader/layers/sanitizer/common.h | 248 +++++++++++ .../sanitizer/device_sanitizer_report.hpp | 116 ++--- .../sanitizer/sanitizer_interceptor.cpp | 404 ++++++------------ .../sanitizer/sanitizer_interceptor.hpp | 47 +- source/loader/layers/sanitizer/ur_sanddi.cpp | 299 +++++++++---- .../layers/sanitizer/ur_sanitizer_layer.cpp | 4 +- .../layers/sanitizer/ur_sanitizer_layer.hpp | 7 +- 7 files changed, 686 insertions(+), 439 deletions(-) create mode 100644 source/loader/layers/sanitizer/common.h diff --git a/source/loader/layers/sanitizer/common.h b/source/loader/layers/sanitizer/common.h new file mode 100644 index 0000000000..cac9280031 --- /dev/null +++ b/source/loader/layers/sanitizer/common.h @@ -0,0 +1,248 @@ +//==---------- sanitizer_interceptor.hpp - Sanitizer interceptor -----------==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include +#include + +#include "ur_ddi.h" + +namespace ur_sanitizer_layer { + +typedef uintptr_t uptr; +typedef unsigned char u8; +typedef unsigned int u32; + +#define ASAN_SHADOW_SCALE 3 +#define ASAN_SHADOW_GRANULARITY (1ULL << ASAN_SHADOW_SCALE) + +inline constexpr bool IsPowerOfTwo(uptr x) { + return (x & (x - 1)) == 0 && x != 0; +} + +inline constexpr uptr RoundUpTo(uptr Size, uptr boundary) { + assert(IsPowerOfTwo(boundary)); + return (Size + boundary - 1) & ~(boundary - 1); +} + +inline constexpr uptr RoundDownTo(uptr x, uptr boundary) { + assert(IsPowerOfTwo(boundary)); + return x & ~(boundary - 1); +} + +inline constexpr bool IsAligned(uptr a, uptr alignment) { + return (a & (alignment - 1)) == 0; +} + +/* +inline uptr LeastSignificantSetBitIndex(uptr x) { + // CHECK_NE(x, 0U); + unsigned long up; +#if !SANITIZER_WINDOWS || defined(__clang__) || defined(__GNUC__) +#ifdef _WIN64 + up = __builtin_ctzll(x); +#else + up = __builtin_ctzl(x); +#endif +#elif defined(_WIN64) + _BitScanForward64(&up, x); +#else + _BitScanForward(&up, x); +#endif + return up; +} + +inline uptr Log2(uptr x) { + // CHECK(IsPowerOfTwo(x)); + return LeastSignificantSetBitIndex(x); +} +*/ + +// Valid redzone sizes are 16, 32, 64, ... 2048, so we encode them in 3 bits. +// We use adaptive redzones: for larger allocation larger redzones are used. +inline constexpr u32 RZLog2Size(u32 rz_log) { + // CHECK_LT(rz_log, 8); + return 16 << rz_log; +} + +/* +static u32 RZSize2Log(u32 rz_size) { + // CHECK_GE(rz_size, 16); + // CHECK_LE(rz_size, 2048); + // CHECK(IsPowerOfTwo(rz_size)); + u32 res = Log2(rz_size) - 4; + // CHECK_EQ(rz_size, RZLog2Size(res)); + return res; +} +*/ + +inline constexpr uptr ComputeRZLog(uptr user_requested_size) { + u32 rz_log = user_requested_size <= 64 - 16 ? 0 + : user_requested_size <= 128 - 32 ? 1 + : user_requested_size <= 512 - 64 ? 2 + : user_requested_size <= 4096 - 128 ? 3 + : user_requested_size <= (1 << 14) - 256 ? 4 + : user_requested_size <= (1 << 15) - 512 ? 5 + : user_requested_size <= (1 << 16) - 1024 ? 6 + : 7; + // u32 hdr_log = RZSize2Log(RoundUpToPowerOfTwo(sizeof(ChunkHeader))); + // u32 min_log = RZSize2Log(atomic_load(&min_redzone, memory_order_acquire)); + // u32 max_log = RZSize2Log(atomic_load(&max_redzone, memory_order_acquire)); + // return Min(Max(rz_log, Max(min_log, hdr_log)), Max(max_log, hdr_log)); + return rz_log; +} + +inline constexpr uptr MemToShadow(uptr Addr, uptr ShadowOffset) { + return ShadowOffset + ((Addr) >> ASAN_SHADOW_SCALE); +} + +static auto getUrResultString = [](ur_result_t Result) { + switch (Result) { + case UR_RESULT_SUCCESS: + return "UR_RESULT_SUCCESS"; + case UR_RESULT_ERROR_INVALID_OPERATION: + return "UR_RESULT_ERROR_INVALID_OPERATION"; + case UR_RESULT_ERROR_INVALID_QUEUE_PROPERTIES: + return "UR_RESULT_ERROR_INVALID_QUEUE_PROPERTIES"; + case UR_RESULT_ERROR_INVALID_QUEUE: + return "UR_RESULT_ERROR_INVALID_QUEUE"; + case UR_RESULT_ERROR_INVALID_VALUE: + return "UR_RESULT_ERROR_INVALID_VALUE"; + case UR_RESULT_ERROR_INVALID_CONTEXT: + return "UR_RESULT_ERROR_INVALID_CONTEXT"; + case UR_RESULT_ERROR_INVALID_PLATFORM: + return "UR_RESULT_ERROR_INVALID_PLATFORM"; + case UR_RESULT_ERROR_INVALID_BINARY: + return "UR_RESULT_ERROR_INVALID_BINARY"; + case UR_RESULT_ERROR_INVALID_PROGRAM: + return "UR_RESULT_ERROR_INVALID_PROGRAM"; + case UR_RESULT_ERROR_INVALID_SAMPLER: + return "UR_RESULT_ERROR_INVALID_SAMPLER"; + case UR_RESULT_ERROR_INVALID_BUFFER_SIZE: + return "UR_RESULT_ERROR_INVALID_BUFFER_SIZE"; + case UR_RESULT_ERROR_INVALID_MEM_OBJECT: + return "UR_RESULT_ERROR_INVALID_MEM_OBJECT"; + case UR_RESULT_ERROR_INVALID_EVENT: + return "UR_RESULT_ERROR_INVALID_EVENT"; + case UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST: + return "UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST"; + case UR_RESULT_ERROR_MISALIGNED_SUB_BUFFER_OFFSET: + return "UR_RESULT_ERROR_MISALIGNED_SUB_BUFFER_OFFSET"; + case UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE: + return "UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE"; + case UR_RESULT_ERROR_COMPILER_NOT_AVAILABLE: + return "UR_RESULT_ERROR_COMPILER_NOT_AVAILABLE"; + case UR_RESULT_ERROR_PROFILING_INFO_NOT_AVAILABLE: + return "UR_RESULT_ERROR_PROFILING_INFO_NOT_AVAILABLE"; + case UR_RESULT_ERROR_DEVICE_NOT_FOUND: + return "UR_RESULT_ERROR_DEVICE_NOT_FOUND"; + case UR_RESULT_ERROR_INVALID_DEVICE: + return "UR_RESULT_ERROR_INVALID_DEVICE"; + case UR_RESULT_ERROR_DEVICE_LOST: + return "UR_RESULT_ERROR_DEVICE_LOST"; + case UR_RESULT_ERROR_DEVICE_REQUIRES_RESET: + return "UR_RESULT_ERROR_DEVICE_REQUIRES_RESET"; + case UR_RESULT_ERROR_DEVICE_IN_LOW_POWER_STATE: + return "UR_RESULT_ERROR_DEVICE_IN_LOW_POWER_STATE"; + case UR_RESULT_ERROR_DEVICE_PARTITION_FAILED: + return "UR_RESULT_ERROR_DEVICE_PARTITION_FAILED"; + case UR_RESULT_ERROR_INVALID_DEVICE_PARTITION_COUNT: + return "UR_RESULT_ERROR_INVALID_DEVICE_PARTITION_COUNT"; + case UR_RESULT_ERROR_INVALID_WORK_ITEM_SIZE: + return "UR_RESULT_ERROR_INVALID_WORK_ITEM_SIZE"; + case UR_RESULT_ERROR_INVALID_WORK_DIMENSION: + return "UR_RESULT_ERROR_INVALID_WORK_DIMENSION"; + case UR_RESULT_ERROR_INVALID_KERNEL_ARGS: + return "UR_RESULT_ERROR_INVALID_KERNEL_ARGS"; + case UR_RESULT_ERROR_INVALID_KERNEL: + return "UR_RESULT_ERROR_INVALID_KERNEL"; + case UR_RESULT_ERROR_INVALID_KERNEL_NAME: + return "UR_RESULT_ERROR_INVALID_KERNEL_NAME"; + case UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX: + return "UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX"; + case UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_SIZE: + return "UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_SIZE"; + case UR_RESULT_ERROR_INVALID_KERNEL_ATTRIBUTE_VALUE: + return "UR_RESULT_ERROR_INVALID_KERNEL_ATTRIBUTE_VALUE"; + case UR_RESULT_ERROR_INVALID_IMAGE_SIZE: + return "UR_RESULT_ERROR_INVALID_IMAGE_SIZE"; + case UR_RESULT_ERROR_INVALID_IMAGE_FORMAT_DESCRIPTOR: + return "UR_RESULT_ERROR_INVALID_IMAGE_FORMAT_DESCRIPTOR"; + case UR_RESULT_ERROR_IMAGE_FORMAT_NOT_SUPPORTED: + return "UR_RESULT_ERROR_IMAGE_FORMAT_NOT_SUPPORTED"; + case UR_RESULT_ERROR_MEM_OBJECT_ALLOCATION_FAILURE: + return "UR_RESULT_ERROR_MEM_OBJECT_ALLOCATION_FAILURE"; + case UR_RESULT_ERROR_INVALID_PROGRAM_EXECUTABLE: + return "UR_RESULT_ERROR_INVALID_PROGRAM_EXECUTABLE"; + case UR_RESULT_ERROR_UNINITIALIZED: + return "UR_RESULT_ERROR_UNINITIALIZED"; + case UR_RESULT_ERROR_OUT_OF_HOST_MEMORY: + return "UR_RESULT_ERROR_OUT_OF_HOST_MEMORY"; + case UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY: + return "UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY"; + case UR_RESULT_ERROR_OUT_OF_RESOURCES: + return "UR_RESULT_ERROR_OUT_OF_RESOURCES"; + case UR_RESULT_ERROR_PROGRAM_BUILD_FAILURE: + return "UR_RESULT_ERROR_PROGRAM_BUILD_FAILURE"; + case UR_RESULT_ERROR_PROGRAM_LINK_FAILURE: + return "UR_RESULT_ERROR_PROGRAM_LINK_FAILURE"; + case UR_RESULT_ERROR_UNSUPPORTED_VERSION: + return "UR_RESULT_ERROR_UNSUPPORTED_VERSION"; + case UR_RESULT_ERROR_UNSUPPORTED_FEATURE: + return "UR_RESULT_ERROR_UNSUPPORTED_FEATURE"; + case UR_RESULT_ERROR_INVALID_ARGUMENT: + return "UR_RESULT_ERROR_INVALID_ARGUMENT"; + case UR_RESULT_ERROR_INVALID_NULL_HANDLE: + return "UR_RESULT_ERROR_INVALID_NULL_HANDLE"; + case UR_RESULT_ERROR_HANDLE_OBJECT_IN_USE: + return "UR_RESULT_ERROR_HANDLE_OBJECT_IN_USE"; + case UR_RESULT_ERROR_INVALID_NULL_POINTER: + return "UR_RESULT_ERROR_INVALID_NULL_POINTER"; + case UR_RESULT_ERROR_INVALID_SIZE: + return "UR_RESULT_ERROR_INVALID_SIZE"; + case UR_RESULT_ERROR_UNSUPPORTED_SIZE: + return "UR_RESULT_ERROR_UNSUPPORTED_SIZE"; + case UR_RESULT_ERROR_UNSUPPORTED_ALIGNMENT: + return "UR_RESULT_ERROR_UNSUPPORTED_ALIGNMENT"; + case UR_RESULT_ERROR_INVALID_SYNCHRONIZATION_OBJECT: + return "UR_RESULT_ERROR_INVALID_SYNCHRONIZATION_OBJECT"; + case UR_RESULT_ERROR_INVALID_ENUMERATION: + return "UR_RESULT_ERROR_INVALID_ENUMERATION"; + case UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION: + return "UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION"; + case UR_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT: + return "UR_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT"; + case UR_RESULT_ERROR_INVALID_NATIVE_BINARY: + return "UR_RESULT_ERROR_INVALID_NATIVE_BINARY"; + case UR_RESULT_ERROR_INVALID_GLOBAL_NAME: + return "UR_RESULT_ERROR_INVALID_GLOBAL_NAME"; + case UR_RESULT_ERROR_INVALID_FUNCTION_NAME: + return "UR_RESULT_ERROR_INVALID_FUNCTION_NAME"; + case UR_RESULT_ERROR_INVALID_GROUP_SIZE_DIMENSION: + return "UR_RESULT_ERROR_INVALID_GROUP_SIZE_DIMENSION"; + case UR_RESULT_ERROR_INVALID_GLOBAL_WIDTH_DIMENSION: + return "UR_RESULT_ERROR_INVALID_GLOBAL_WIDTH_DIMENSION"; + case UR_RESULT_ERROR_PROGRAM_UNLINKED: + return "UR_RESULT_ERROR_PROGRAM_UNLINKED"; + case UR_RESULT_ERROR_OVERLAPPING_REGIONS: + return "UR_RESULT_ERROR_OVERLAPPING_REGIONS"; + case UR_RESULT_ERROR_INVALID_HOST_PTR: + return "UR_RESULT_ERROR_INVALID_HOST_PTR"; + case UR_RESULT_ERROR_INVALID_USM_SIZE: + return "UR_RESULT_ERROR_INVALID_USM_SIZE"; + case UR_RESULT_ERROR_OBJECT_ALLOCATION_FAILURE: + return "UR_RESULT_ERROR_OBJECT_ALLOCATION_FAILURE"; + case UR_RESULT_ERROR_ADAPTER_SPECIFIC: + return "UR_RESULT_ERROR_ADAPTER_SPECIFIC"; + default: + return "UR_RESULT_ERROR_UNKNOWN"; + } +}; + +} // namespace ur_sanitizer_layer diff --git a/source/loader/layers/sanitizer/device_sanitizer_report.hpp b/source/loader/layers/sanitizer/device_sanitizer_report.hpp index fd31a61e82..e65aacd5e8 100644 --- a/source/loader/layers/sanitizer/device_sanitizer_report.hpp +++ b/source/loader/layers/sanitizer/device_sanitizer_report.hpp @@ -1,78 +1,90 @@ +//==---------- sanitizer_interceptor.hpp - Sanitizer interceptor -----------==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + #pragma once #include +namespace ur_sanitizer_layer { + enum class DeviceSanitizerErrorType : int32_t { - OUT_OF_BOUND, - MISALIGNED, - USE_AFTER_FREE, - OUT_OF_SHADOW_BOUND, - UNKNOWN + OUT_OF_BOUND, + MISALIGNED, + USE_AFTER_FREE, + OUT_OF_SHADOW_BOUND, + UNKNOWN }; enum class DeviceSanitizerMemoryType : int32_t { - USM_DEVICE, - USM_HOST, - USM_SHARED, - LOCAL, - PRIVATE, - UNKNOWN + USM_DEVICE, + USM_HOST, + USM_SHARED, + LOCAL, + PRIVATE, + UNKNOWN }; // NOTE Layout of this structure should be aligned with the one in // sycl/include/sycl/detail/device_sanitizer_report.hpp struct DeviceSanitizerReport { - int Flag = 0; + int Flag = 0; - char File[256 + 1] = ""; - char Func[128 + 1] = ""; + char File[256 + 1] = ""; + char Func[128 + 1] = ""; - int32_t Line = 0; + int32_t Line = 0; - uint64_t GID0 = 0; - uint64_t GID1 = 0; - uint64_t GID2 = 0; + uint64_t GID0 = 0; + uint64_t GID1 = 0; + uint64_t GID2 = 0; - uint64_t LID0 = 0; - uint64_t LID1 = 0; - uint64_t LID2 = 0; + uint64_t LID0 = 0; + uint64_t LID1 = 0; + uint64_t LID2 = 0; - bool IsWrite = false; - uint32_t AccessSize = 0; - DeviceSanitizerMemoryType MemoryType; - DeviceSanitizerErrorType ErrorType; + bool IsWrite = false; + uint32_t AccessSize = 0; + DeviceSanitizerMemoryType MemoryType; + DeviceSanitizerErrorType ErrorType; - bool IsRecover = false; + bool IsRecover = false; }; const char *DeviceSanitizerFormat(DeviceSanitizerMemoryType MemoryType) { - switch (MemoryType) { - case DeviceSanitizerMemoryType::USM_DEVICE: - return "USM Device Memory"; - case DeviceSanitizerMemoryType::USM_HOST: - return "USM Host Memory"; - case DeviceSanitizerMemoryType::USM_SHARED: - return "USM Shared Memory"; - case DeviceSanitizerMemoryType::LOCAL: - return "Local Memory"; - case DeviceSanitizerMemoryType::PRIVATE: - return "Private Memory"; - default: - return "Unknown Memory"; - } + switch (MemoryType) { + case DeviceSanitizerMemoryType::USM_DEVICE: + return "USM Device Memory"; + case DeviceSanitizerMemoryType::USM_HOST: + return "USM Host Memory"; + case DeviceSanitizerMemoryType::USM_SHARED: + return "USM Shared Memory"; + case DeviceSanitizerMemoryType::LOCAL: + return "Local Memory"; + case DeviceSanitizerMemoryType::PRIVATE: + return "Private Memory"; + default: + return "Unknown Memory"; + } } const char *DeviceSanitizerFormat(DeviceSanitizerErrorType ErrorType) { - switch (ErrorType) { - case DeviceSanitizerErrorType::OUT_OF_BOUND: - return "out-of-bound-access"; - case DeviceSanitizerErrorType::MISALIGNED: - return "misaligned-access"; - case DeviceSanitizerErrorType::USE_AFTER_FREE: - return "use-after-free"; - case DeviceSanitizerErrorType::OUT_OF_SHADOW_BOUND: - return "out-of-shadow-bound-access"; - default: - return "unknown-error"; - } + switch (ErrorType) { + case DeviceSanitizerErrorType::OUT_OF_BOUND: + return "out-of-bound-access"; + case DeviceSanitizerErrorType::MISALIGNED: + return "misaligned-access"; + case DeviceSanitizerErrorType::USE_AFTER_FREE: + return "use-after-free"; + case DeviceSanitizerErrorType::OUT_OF_SHADOW_BOUND: + return "out-of-shadow-bound-access"; + default: + return "unknown-error"; + } } + +} // namespace ur_sanitizer_layer diff --git a/source/loader/layers/sanitizer/sanitizer_interceptor.cpp b/source/loader/layers/sanitizer/sanitizer_interceptor.cpp index 490998fce5..236830322e 100644 --- a/source/loader/layers/sanitizer/sanitizer_interceptor.cpp +++ b/source/loader/layers/sanitizer/sanitizer_interceptor.cpp @@ -8,24 +8,26 @@ #include "sanitizer_interceptor.hpp" #include "device_sanitizer_report.hpp" +#include "ur_sanitizer_layer.hpp" -#include -#include #include #include #include -#define ASAN_SHADOW_SCALE 3 -#define ASAN_SHADOW_GRANULARITY (1ULL << ASAN_SHADOW_SCALE) +namespace ur_sanitizer_layer { + +namespace { // These magic values are written to shadow for better error // reporting. const int kUsmDeviceRedzoneMagic = 0x81; const int kUsmHostRedzoneMagic = 0x82; const int kUsmSharedRedzoneMagic = 0x83; -const int kUsmDeviceDeallocatedMagic = 0x84; -const int kUsmHostDeallocatedMagic = 0x85; -const int kUsmSharedDeallocatedMagic = 0x86; +const int kMemBufferRedzoneMagic = 0x84; + +const int kUsmDeviceDeallocatedMagic = 0x91; +const int kUsmHostDeallocatedMagic = 0x92; +const int kUsmSharedDeallocatedMagic = 0x93; // Same with Asan Stack const int kPrivateLeftRedzoneMagic = 0xf1; @@ -56,241 +58,14 @@ const auto kSPIR_AsanShadowMemoryGlobalEnd = "__AsanShadowMemoryGlobalEnd"; const auto kSPIR_DeviceSanitizerReportMem = "__DeviceSanitizerReportMem"; -namespace { - DeviceSanitizerReport SPIR_DeviceSanitizerReportMem; -inline constexpr bool IsPowerOfTwo(uptr x) { - return (x & (x - 1)) == 0 && x != 0; -} - -inline constexpr uptr RoundUpTo(uptr Size, uptr boundary) { - assert(IsPowerOfTwo(boundary)); - return (Size + boundary - 1) & ~(boundary - 1); -} - -inline constexpr uptr RoundDownTo(uptr x, uptr boundary) { - assert(IsPowerOfTwo(boundary)); - return x & ~(boundary - 1); -} - -inline constexpr bool IsAligned(uptr a, uptr alignment) { - return (a & (alignment - 1)) == 0; -} - -/* -inline uptr LeastSignificantSetBitIndex(uptr x) { - // CHECK_NE(x, 0U); - unsigned long up; -#if !SANITIZER_WINDOWS || defined(__clang__) || defined(__GNUC__) -#ifdef _WIN64 - up = __builtin_ctzll(x); -#else - up = __builtin_ctzl(x); -#endif -#elif defined(_WIN64) - _BitScanForward64(&up, x); -#else - _BitScanForward(&up, x); -#endif - return up; -} - -inline uptr Log2(uptr x) { - // CHECK(IsPowerOfTwo(x)); - return LeastSignificantSetBitIndex(x); -} -*/ - -// Valid redzone sizes are 16, 32, 64, ... 2048, so we encode them in 3 bits. -// We use adaptive redzones: for larger allocation larger redzones are used. -static u32 RZLog2Size(u32 rz_log) { - // CHECK_LT(rz_log, 8); - return 16 << rz_log; -} - -/* -static u32 RZSize2Log(u32 rz_size) { - // CHECK_GE(rz_size, 16); - // CHECK_LE(rz_size, 2048); - // CHECK(IsPowerOfTwo(rz_size)); - u32 res = Log2(rz_size) - 4; - // CHECK_EQ(rz_size, RZLog2Size(res)); - return res; -} -*/ - -uptr ComputeRZLog(uptr user_requested_size) { - u32 rz_log = user_requested_size <= 64 - 16 ? 0 - : user_requested_size <= 128 - 32 ? 1 - : user_requested_size <= 512 - 64 ? 2 - : user_requested_size <= 4096 - 128 ? 3 - : user_requested_size <= (1 << 14) - 256 ? 4 - : user_requested_size <= (1 << 15) - 512 ? 5 - : user_requested_size <= (1 << 16) - 1024 ? 6 - : 7; - // u32 hdr_log = RZSize2Log(RoundUpToPowerOfTwo(sizeof(ChunkHeader))); - // u32 min_log = RZSize2Log(atomic_load(&min_redzone, memory_order_acquire)); - // u32 max_log = RZSize2Log(atomic_load(&max_redzone, memory_order_acquire)); - // return Min(Max(rz_log, Max(min_log, hdr_log)), Max(max_log, hdr_log)); - return rz_log; -} - -inline constexpr uptr MemToShadow(uptr Addr, uptr ShadowOffset) { - return ShadowOffset + ((Addr) >> ASAN_SHADOW_SCALE); -} - -static auto getUrResultString = [](ur_result_t Result) { - switch (Result) { - case UR_RESULT_SUCCESS: - return "UR_RESULT_SUCCESS"; - case UR_RESULT_ERROR_INVALID_OPERATION: - return "UR_RESULT_ERROR_INVALID_OPERATION"; - case UR_RESULT_ERROR_INVALID_QUEUE_PROPERTIES: - return "UR_RESULT_ERROR_INVALID_QUEUE_PROPERTIES"; - case UR_RESULT_ERROR_INVALID_QUEUE: - return "UR_RESULT_ERROR_INVALID_QUEUE"; - case UR_RESULT_ERROR_INVALID_VALUE: - return "UR_RESULT_ERROR_INVALID_VALUE"; - case UR_RESULT_ERROR_INVALID_CONTEXT: - return "UR_RESULT_ERROR_INVALID_CONTEXT"; - case UR_RESULT_ERROR_INVALID_PLATFORM: - return "UR_RESULT_ERROR_INVALID_PLATFORM"; - case UR_RESULT_ERROR_INVALID_BINARY: - return "UR_RESULT_ERROR_INVALID_BINARY"; - case UR_RESULT_ERROR_INVALID_PROGRAM: - return "UR_RESULT_ERROR_INVALID_PROGRAM"; - case UR_RESULT_ERROR_INVALID_SAMPLER: - return "UR_RESULT_ERROR_INVALID_SAMPLER"; - case UR_RESULT_ERROR_INVALID_BUFFER_SIZE: - return "UR_RESULT_ERROR_INVALID_BUFFER_SIZE"; - case UR_RESULT_ERROR_INVALID_MEM_OBJECT: - return "UR_RESULT_ERROR_INVALID_MEM_OBJECT"; - case UR_RESULT_ERROR_INVALID_EVENT: - return "UR_RESULT_ERROR_INVALID_EVENT"; - case UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST: - return "UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST"; - case UR_RESULT_ERROR_MISALIGNED_SUB_BUFFER_OFFSET: - return "UR_RESULT_ERROR_MISALIGNED_SUB_BUFFER_OFFSET"; - case UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE: - return "UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE"; - case UR_RESULT_ERROR_COMPILER_NOT_AVAILABLE: - return "UR_RESULT_ERROR_COMPILER_NOT_AVAILABLE"; - case UR_RESULT_ERROR_PROFILING_INFO_NOT_AVAILABLE: - return "UR_RESULT_ERROR_PROFILING_INFO_NOT_AVAILABLE"; - case UR_RESULT_ERROR_DEVICE_NOT_FOUND: - return "UR_RESULT_ERROR_DEVICE_NOT_FOUND"; - case UR_RESULT_ERROR_INVALID_DEVICE: - return "UR_RESULT_ERROR_INVALID_DEVICE"; - case UR_RESULT_ERROR_DEVICE_LOST: - return "UR_RESULT_ERROR_DEVICE_LOST"; - case UR_RESULT_ERROR_DEVICE_REQUIRES_RESET: - return "UR_RESULT_ERROR_DEVICE_REQUIRES_RESET"; - case UR_RESULT_ERROR_DEVICE_IN_LOW_POWER_STATE: - return "UR_RESULT_ERROR_DEVICE_IN_LOW_POWER_STATE"; - case UR_RESULT_ERROR_DEVICE_PARTITION_FAILED: - return "UR_RESULT_ERROR_DEVICE_PARTITION_FAILED"; - case UR_RESULT_ERROR_INVALID_DEVICE_PARTITION_COUNT: - return "UR_RESULT_ERROR_INVALID_DEVICE_PARTITION_COUNT"; - case UR_RESULT_ERROR_INVALID_WORK_ITEM_SIZE: - return "UR_RESULT_ERROR_INVALID_WORK_ITEM_SIZE"; - case UR_RESULT_ERROR_INVALID_WORK_DIMENSION: - return "UR_RESULT_ERROR_INVALID_WORK_DIMENSION"; - case UR_RESULT_ERROR_INVALID_KERNEL_ARGS: - return "UR_RESULT_ERROR_INVALID_KERNEL_ARGS"; - case UR_RESULT_ERROR_INVALID_KERNEL: - return "UR_RESULT_ERROR_INVALID_KERNEL"; - case UR_RESULT_ERROR_INVALID_KERNEL_NAME: - return "UR_RESULT_ERROR_INVALID_KERNEL_NAME"; - case UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX: - return "UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX"; - case UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_SIZE: - return "UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_SIZE"; - case UR_RESULT_ERROR_INVALID_KERNEL_ATTRIBUTE_VALUE: - return "UR_RESULT_ERROR_INVALID_KERNEL_ATTRIBUTE_VALUE"; - case UR_RESULT_ERROR_INVALID_IMAGE_SIZE: - return "UR_RESULT_ERROR_INVALID_IMAGE_SIZE"; - case UR_RESULT_ERROR_INVALID_IMAGE_FORMAT_DESCRIPTOR: - return "UR_RESULT_ERROR_INVALID_IMAGE_FORMAT_DESCRIPTOR"; - case UR_RESULT_ERROR_IMAGE_FORMAT_NOT_SUPPORTED: - return "UR_RESULT_ERROR_IMAGE_FORMAT_NOT_SUPPORTED"; - case UR_RESULT_ERROR_MEM_OBJECT_ALLOCATION_FAILURE: - return "UR_RESULT_ERROR_MEM_OBJECT_ALLOCATION_FAILURE"; - case UR_RESULT_ERROR_INVALID_PROGRAM_EXECUTABLE: - return "UR_RESULT_ERROR_INVALID_PROGRAM_EXECUTABLE"; - case UR_RESULT_ERROR_UNINITIALIZED: - return "UR_RESULT_ERROR_UNINITIALIZED"; - case UR_RESULT_ERROR_OUT_OF_HOST_MEMORY: - return "UR_RESULT_ERROR_OUT_OF_HOST_MEMORY"; - case UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY: - return "UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY"; - case UR_RESULT_ERROR_OUT_OF_RESOURCES: - return "UR_RESULT_ERROR_OUT_OF_RESOURCES"; - case UR_RESULT_ERROR_PROGRAM_BUILD_FAILURE: - return "UR_RESULT_ERROR_PROGRAM_BUILD_FAILURE"; - case UR_RESULT_ERROR_PROGRAM_LINK_FAILURE: - return "UR_RESULT_ERROR_PROGRAM_LINK_FAILURE"; - case UR_RESULT_ERROR_UNSUPPORTED_VERSION: - return "UR_RESULT_ERROR_UNSUPPORTED_VERSION"; - case UR_RESULT_ERROR_UNSUPPORTED_FEATURE: - return "UR_RESULT_ERROR_UNSUPPORTED_FEATURE"; - case UR_RESULT_ERROR_INVALID_ARGUMENT: - return "UR_RESULT_ERROR_INVALID_ARGUMENT"; - case UR_RESULT_ERROR_INVALID_NULL_HANDLE: - return "UR_RESULT_ERROR_INVALID_NULL_HANDLE"; - case UR_RESULT_ERROR_HANDLE_OBJECT_IN_USE: - return "UR_RESULT_ERROR_HANDLE_OBJECT_IN_USE"; - case UR_RESULT_ERROR_INVALID_NULL_POINTER: - return "UR_RESULT_ERROR_INVALID_NULL_POINTER"; - case UR_RESULT_ERROR_INVALID_SIZE: - return "UR_RESULT_ERROR_INVALID_SIZE"; - case UR_RESULT_ERROR_UNSUPPORTED_SIZE: - return "UR_RESULT_ERROR_UNSUPPORTED_SIZE"; - case UR_RESULT_ERROR_UNSUPPORTED_ALIGNMENT: - return "UR_RESULT_ERROR_UNSUPPORTED_ALIGNMENT"; - case UR_RESULT_ERROR_INVALID_SYNCHRONIZATION_OBJECT: - return "UR_RESULT_ERROR_INVALID_SYNCHRONIZATION_OBJECT"; - case UR_RESULT_ERROR_INVALID_ENUMERATION: - return "UR_RESULT_ERROR_INVALID_ENUMERATION"; - case UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION: - return "UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION"; - case UR_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT: - return "UR_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT"; - case UR_RESULT_ERROR_INVALID_NATIVE_BINARY: - return "UR_RESULT_ERROR_INVALID_NATIVE_BINARY"; - case UR_RESULT_ERROR_INVALID_GLOBAL_NAME: - return "UR_RESULT_ERROR_INVALID_GLOBAL_NAME"; - case UR_RESULT_ERROR_INVALID_FUNCTION_NAME: - return "UR_RESULT_ERROR_INVALID_FUNCTION_NAME"; - case UR_RESULT_ERROR_INVALID_GROUP_SIZE_DIMENSION: - return "UR_RESULT_ERROR_INVALID_GROUP_SIZE_DIMENSION"; - case UR_RESULT_ERROR_INVALID_GLOBAL_WIDTH_DIMENSION: - return "UR_RESULT_ERROR_INVALID_GLOBAL_WIDTH_DIMENSION"; - case UR_RESULT_ERROR_PROGRAM_UNLINKED: - return "UR_RESULT_ERROR_PROGRAM_UNLINKED"; - case UR_RESULT_ERROR_OVERLAPPING_REGIONS: - return "UR_RESULT_ERROR_OVERLAPPING_REGIONS"; - case UR_RESULT_ERROR_INVALID_HOST_PTR: - return "UR_RESULT_ERROR_INVALID_HOST_PTR"; - case UR_RESULT_ERROR_INVALID_USM_SIZE: - return "UR_RESULT_ERROR_INVALID_USM_SIZE"; - case UR_RESULT_ERROR_OBJECT_ALLOCATION_FAILURE: - return "UR_RESULT_ERROR_OBJECT_ALLOCATION_FAILURE"; - case UR_RESULT_ERROR_ADAPTER_SPECIFIC: - return "UR_RESULT_ERROR_ADAPTER_SPECIFIC"; - default: - return "UR_RESULT_ERROR_UNKNOWN"; - } -}; - } // namespace ur_result_t SanitizerInterceptor::allocateMemory( ur_context_handle_t Context, ur_device_handle_t Device, const ur_usm_desc_t *Properties, ur_usm_pool_handle_t Pool, size_t Size, void **ResultPtr, USMMemoryType Type) { - (void)Context; - auto Alignment = Properties->align; assert(Alignment == 0 || IsPowerOfTwo(Alignment)); @@ -315,15 +90,10 @@ ur_result_t SanitizerInterceptor::allocateMemory( } // Calcuate Size + RZSize - uptr rz_log = ComputeRZLog(Size); - uptr rz_size = RZLog2Size(rz_log); - uptr rounded_size = RoundUpTo(Size, Alignment); - uptr NeededSize = rounded_size + rz_size * 2; - - std::cerr << "allocateMemory:" - << "\n user_size: " << Size << "\n rz_size: " << rz_size - << "\n rounded_size: " << rounded_size - << "\n NeededSize: " << NeededSize << std::endl; + uptr RZLog = ComputeRZLog(Size); + uptr RZSize = RZLog2Size(RZLog); + uptr RoundedSize = RoundUpTo(Size, Alignment); + uptr NeededSize = RoundedSize + RZSize * 2; void *Allocated = nullptr; ur_result_t Result = UR_RESULT_SUCCESS; @@ -346,7 +116,7 @@ ur_result_t SanitizerInterceptor::allocateMemory( // Enqueue Shadow Memory Init uptr AllocBegin = reinterpret_cast(Allocated); uptr AllocEnd = AllocBegin + NeededSize; - uptr UserBegin = AllocBegin + rz_size; + uptr UserBegin = AllocBegin + RZSize; if (!IsAligned(UserBegin, Alignment)) { UserBegin = RoundUpTo(UserBegin, Alignment); } @@ -356,7 +126,7 @@ ur_result_t SanitizerInterceptor::allocateMemory( *ResultPtr = reinterpret_cast(UserBegin); auto MemoryInfo = - AllocatedMemoryInfo{AllocBegin, UserBegin, UserEnd, NeededSize, Type}; + USMMemoryInfo{AllocBegin, UserBegin, UserEnd, NeededSize, Type}; if (Device) { MemoryInfo.Devices.emplace(Device); } @@ -375,12 +145,11 @@ ur_result_t SanitizerInterceptor::allocateMemory( { std::lock_guard Guard(ContextInfo.Mutex); ContextInfo.AllocatedAddressesMap[AllocBegin] = MemoryInfo; - std::cout << "AllocatedAddressesMap: " << (void *)AllocBegin << "\n"; } - std::cout << "AllocInfos: " << (void *)AllocBegin << " " - << (void *)UserBegin << "-" << (void *)UserEnd << " " - << NeededSize << " " << (void *)Type << std::endl; + context.logger.info("AllocInfos:\n AllocBegin: {}\n User: {}-{}\n " + "NeededSize: {}\nType: {}", + AllocBegin, UserBegin, UserEnd, NeededSize, Type); return UR_RESULT_SUCCESS; } @@ -390,7 +159,7 @@ ur_result_t SanitizerInterceptor::releaseMemory(ur_context_handle_t Context, auto &ContextInfo = getContextInfo(Context); assert(ContextInfo.Init); - std::cerr << "releaseMemory: " << Ptr << "\n"; + context.logger.debug("ReleaseMemory: {}", Ptr); std::lock_guard Guard(ContextInfo.Mutex); auto Addr = (uptr)Ptr; @@ -398,8 +167,9 @@ ur_result_t SanitizerInterceptor::releaseMemory(ur_context_handle_t Context, auto AddressInfoIt = ContextInfo.AllocatedAddressesMap.upper_bound((uptr)Addr); if (AddressInfoIt == ContextInfo.AllocatedAddressesMap.begin()) { - std::cerr << "ERROR: releaseMemory failed! AllocatedAddressesMap\n"; - return UR_RESULT_SUCCESS; + context.logger.error( + "Can't find release pointer({}) in AllocatedAddressesMap", Ptr); + return UR_RESULT_ERROR_INVALID_ARGUMENT; } --AddressInfoIt; auto &AddressInfo = AddressInfoIt->second; @@ -407,7 +177,9 @@ ur_result_t SanitizerInterceptor::releaseMemory(ur_context_handle_t Context, << AddressInfo.UserBegin << "\n"; if (Addr != AddressInfo.UserBegin) { std::cerr << "ERROR: releaseMemory failed! UserBegin\n"; - return UR_RESULT_SUCCESS; + context.logger.error("Release pointer({}) is not match to {}", Ptr, + AddressInfo.UserBegin); + return UR_RESULT_ERROR_INVALID_ARGUMENT; } // TODO: Update shadow memory @@ -532,8 +304,8 @@ void SanitizerInterceptor::checkSanitizerError(ur_kernel_handle_t Kernel) { checkSanitizerReport(KernelName.c_str()); } -bool SanitizerInterceptor::updateHostShadowMemory( - ur_context_handle_t Context, AllocatedMemoryInfo AllocInfo) { +bool SanitizerInterceptor::updateHostShadowMemory(ur_context_handle_t Context, + USMMemoryInfo AllocInfo) { auto &ContextInfo = getContextInfo(Context); auto ShadowOffset = ContextInfo.HostShadowOffset; @@ -586,7 +358,7 @@ SanitizerInterceptor::piextMemAllocShadow(ur_context_handle_t Context, DeviceInfo.ShadowOffset = 0x00007fff7fffULL; DeviceInfo.ShadowOffsetEnd = 0x10007fff7fffULL; } else if (DeviceInfo.Type == UR_DEVICE_TYPE_GPU) { - /// SHADOW MAPPING (PVC, with CPU 47bit) + /// SHADOW MEMORY MAPPING (PVC, with CPU 47bit) /// Host/Shared USM : 0x0 ~ 0x0fff_ffff_ffff /// ? : 0x1000_0000_0000 ~ 0x1fff_ffff_ffff /// Device USM : 0x2000_0000_0000 ~ 0x3fff_ffff_ffff @@ -596,9 +368,10 @@ SanitizerInterceptor::piextMemAllocShadow(ur_context_handle_t Context, auto URes = m_Dditable.VirtualMem.pfnReserve( Context, nullptr, SHADOW_SIZE, (void **)&DeviceInfo.ShadowOffset); if (URes != UR_RESULT_SUCCESS) { - printf("urVirtualMemReserve(): %s\n", getUrResultString(URes)); + context.logger.error("urVirtualMemReserve: {}", + getUrResultString(URes)); + return URes; } - assert(URes == UR_RESULT_SUCCESS); DeviceInfo.ShadowOffsetEnd = DeviceInfo.ShadowOffset + SHADOW_SIZE; } else { @@ -644,22 +417,23 @@ ur_result_t SanitizerInterceptor::piextEnqueueMemSetShadow( auto URes = m_Dditable.PhysicalMem.pfnCreate( Context, Device, PageSize, &Desc, &PhysicalMem); if (URes != UR_RESULT_SUCCESS) { - printf(" zePhysicalMemCreate(): %s\n", - getUrResultString(URes)); + context.logger.error("zePhysicalMemCreate(): {}", + getUrResultString(URes)); + return URes; } - assert(URes == UR_RESULT_SUCCESS); } - printf(" zeVirtualMemMap: %p ~ %p\n", (void *)MappedPtr, - (void *)(MappedPtr + PageSize - 1)); + context.logger.debug("zeVirtualMemMap: {} ~ {}", + (void *)MappedPtr, + (void *)(MappedPtr + PageSize - 1)); // FIXME: No flag to check the failed reason is VA is already mapped auto URes = m_Dditable.VirtualMem.pfnMap( Context, (void *)MappedPtr, PageSize, PhysicalMem, 0, UR_VIRTUAL_MEM_ACCESS_FLAG_READ_WRITE); if (URes != UR_RESULT_SUCCESS) { - printf(" zeVirtualMemMap(): %s\n", - getUrResultString(URes)); + context.logger.debug(" zeVirtualMemMap(): %s\n", + getUrResultString(URes)); } // Initialize to zero @@ -674,10 +448,10 @@ ur_result_t SanitizerInterceptor::piextEnqueueMemSetShadow( Queue, (void *)MappedPtr, 1, Pattern, PageSize, NumEventsInWaitList, EventsWaitList, Event); if (URes != UR_RESULT_SUCCESS) { - printf(" urEnqueueUSMFill(): %s\n", - getUrResultString(URes)); + context.logger.error("urEnqueueUSMFill(): {}", + getUrResultString(URes)); + return URes; } - assert(URes == UR_RESULT_SUCCESS); NumEventsInWaitList = 1; EventsWaitList = Event; @@ -691,9 +465,10 @@ ur_result_t SanitizerInterceptor::piextEnqueueMemSetShadow( (ShadowEnd - ShadowBegin + 1), NumEventsInWaitList, EventsWaitList, Event); if (URes != UR_RESULT_SUCCESS) { - printf(" urEnqueueUSMFill(): %s\n", getUrResultString(URes)); + context.logger.error("urEnqueueUSMFill(): {}", + getUrResultString(URes)); + return URes; } - assert(URes == UR_RESULT_SUCCESS); } else { assert(false && "Unsupport device type"); } @@ -713,7 +488,7 @@ ur_result_t SanitizerInterceptor::enqueuePoisonShadow( void SanitizerInterceptor::enqueueAllocInfo(ur_context_handle_t Context, ur_device_handle_t Device, ur_queue_handle_t Queue, - AllocatedMemoryInfo &AllocInfo, + USMMemoryInfo &AllocInfo, ur_event_handle_t &LastEvent) { // Init zero auto Res = @@ -840,8 +615,9 @@ void SanitizerInterceptor::initDevice(ur_context_handle_t Context, Result = piextMemAllocShadow(Context, Device); assert(Result == UR_RESULT_SUCCESS); - std::cout << "Device ShadowOffset: " << (void *)DeviceInfo.ShadowOffset - << " - " << (void *)DeviceInfo.ShadowOffsetEnd << "\n"; + context.logger.info("Device ShadowOffset: {} - {}", + (void *)DeviceInfo.ShadowOffset, + (void *)DeviceInfo.ShadowOffsetEnd); DeviceInfo.Init = true; } @@ -876,7 +652,6 @@ bool SanitizerInterceptor::initKernel(ur_queue_handle_t Queue, std::lock_guard QueueGuard(QueueInfo.Mutex); ur_event_handle_t LastEvent = QueueInfo.LastEvent; - bool Res = true; do { // Set global variable to program auto EnqueueWriteGlobal = [&](const char *Name, const void *Value) { @@ -888,8 +663,8 @@ bool SanitizerInterceptor::initKernel(ur_queue_handle_t Queue, Queue, Program, Name, false, sizeof(uptr), 0, Value, NumEvents, EventsList, &NewEvent); if (Result != UR_RESULT_SUCCESS) { - std::cerr << "WARNING: Device Global Write Failed [" << Name - << "] " << Result << std::endl; + context.logger.warning("Device Global Write Failed[{}]: {}", + Name, getUrResultString(Result)); return false; } LastEvent = NewEvent; @@ -903,9 +678,80 @@ bool SanitizerInterceptor::initKernel(ur_queue_handle_t Queue, &DeviceInfo.ShadowOffsetEnd); } while (false); - assert(Res && "Init Kernel Failed"); - QueueInfo.LastEvent = LastEvent; - return Res; + return true; } + +ur_result_t SanitizerInterceptor::createMemoryBuffer( + ur_context_handle_t Context, ur_mem_flags_t Flags, size_t Size, + const ur_buffer_properties_t *Properties, ur_mem_handle_t *Buffer) { + auto &ContextInfo = getContextInfo(Context); + if (!ContextInfo.Init) { + initContext(Context); + } + + constexpr size_t Alignment = 8; + + // Calcuate Size + RZSize + const uptr RZLog = ComputeRZLog(Size); + const uptr RZSize = RZLog2Size(RZLog); + const uptr RoundedSize = RoundUpTo(Size, Alignment); + const uptr NeededSize = RoundedSize + RZSize * 2; + + auto Result = m_Dditable.Mem.pfnBufferCreate(Context, Flags, NeededSize, + Properties, Buffer); + if (Result != UR_RESULT_SUCCESS) { + return Result; + } + + // Get Native Handle + ur_native_handle_t NativeHandle{}; + Result = m_Dditable.Mem.pfnGetNativeHandle(*Buffer, &NativeHandle); + if (Result != UR_RESULT_SUCCESS) { + return Result; + } + void *Allocated = (void *)NativeHandle; + + ur_device_handle_t Device; + Result = m_Dditable.USM.pfnGetMemAllocInfo(Context, Allocated, UR_USM_ALLOC_INFO_DEVICE, sizeof(Device), &Device, nullptr); + + uptr AllocBegin = reinterpret_cast(Allocated); + uptr AllocEnd = AllocBegin + NeededSize; + uptr UserBegin = AllocBegin + RZSize; + if (!IsAligned(UserBegin, Alignment)) { + UserBegin = RoundUpTo(UserBegin, Alignment); + } + uptr UserEnd = UserBegin + Size; + assert(UserEnd <= AllocEnd); + + auto MemoryInfo = USMMemoryInfo{AllocBegin, UserBegin, UserEnd, NeededSize, + USMMemoryType::DEVICE}; + MemoryInfo.Devices.emplace(Device); + + // Update Shadow Memory + auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); + std::lock_guard Guard(DeviceInfo.Mutex); + DeviceInfo.AllocInfos.emplace_back(MemoryInfo); + + // Save into AllocatedAddressesMap for releasing + { + std::lock_guard Guard(ContextInfo.Mutex); + ContextInfo.AllocatedAddressesMap[AllocBegin] = MemoryInfo; + } + + // Create a subbuffer which is surrounded by red zone + ur_buffer_region_t BufferRegion{UR_STRUCTURE_TYPE_BUFFER_REGION, nullptr, UserBegin - AllocBegin, Size}; + ur_mem_handle_t SubBuffer; + Result = m_Dditable.Mem.pfnBufferPartition(*Buffer, Flags, + UR_BUFFER_CREATE_TYPE_REGION, + &BufferRegion, &SubBuffer); + if (Result != UR_RESULT_SUCCESS) { + return Result; + } + + *Buffer = SubBuffer; + return UR_RESULT_SUCCESS; +} + +} // namespace ur_sanitizer_layer diff --git a/source/loader/layers/sanitizer/sanitizer_interceptor.hpp b/source/loader/layers/sanitizer/sanitizer_interceptor.hpp index 7c15cf153b..97b0c794d0 100644 --- a/source/loader/layers/sanitizer/sanitizer_interceptor.hpp +++ b/source/loader/layers/sanitizer/sanitizer_interceptor.hpp @@ -8,10 +8,9 @@ #pragma once -#include "ur_ddi.h" +#include "common.h" #include -#include #include #include #include @@ -21,14 +20,13 @@ #include #include -typedef uintptr_t uptr; -typedef unsigned char u8; -typedef unsigned int u32; +namespace ur_sanitizer_layer { enum USMMemoryType { DEVICE, SHARE, HOST, + MEM_BUFFER }; class SanitizerInterceptor { @@ -49,15 +47,20 @@ class SanitizerInterceptor { void postLaunchKernel(ur_kernel_handle_t Kernel, ur_queue_handle_t Queue, ur_event_handle_t *Event, bool SetCallback = true); void checkSanitizerError(ur_kernel_handle_t Kernel); + ur_result_t createMemoryBuffer(ur_context_handle_t Context, + ur_mem_flags_t Flags, size_t Size, + const ur_buffer_properties_t *Properties, + ur_mem_handle_t *Buffer); private: - struct AllocatedMemoryInfo { + struct USMMemoryInfo { uptr AllocBegin; uptr UserBegin; uptr UserEnd; size_t AllocSize; USMMemoryType Type; - std::unordered_set Devices; + std::unordered_set + Devices; // host USM have a lot of devices }; struct DeviceInfo { @@ -69,25 +72,35 @@ class SanitizerInterceptor { uptr ShadowOffset; uptr ShadowOffsetEnd; - std::vector AllocInfos; + std::vector AllocInfos; std::mutex Mutex; // Lock Init & InitPool & AllocInfos }; + struct MemBufferInfo { + ur_mem_handle_t Buffer; + USMMemoryInfo AllocInfo; + size_t Size; + size_t RZSize; + }; + enum class DeviceType { CPU, GPU_PVC, GPU_DG2 }; struct ContextInfo { bool Init = false; - /// AllocatedMemoryInfo.AllocBegin => AllocatedMemoryInfo + /// USMMemoryInfo.AllocBegin => USMMemoryInfo /// /// Use AllocBegin as key can help to detect underflow pointer - std::map AllocatedAddressesMap; + std::map AllocatedAddressesMap; - std::vector AllocHostInfos; + std::vector AllocHostInfos; /// Each context is able to contain multiple devices std::unordered_map DeviceMap; + /// + std::unordered_map MemBufferMap; + std::mutex Mutex; // Lock Init and Maps uptr HostShadowOffset; @@ -96,10 +109,14 @@ class SanitizerInterceptor { std::lock_guard Guard(Mutex); return DeviceMap[Device]; } - AllocatedMemoryInfo &getAllocatedMemoryInfo(uptr Address) { + USMMemoryInfo &getUSMMemoryInfo(uptr Address) { std::lock_guard Guard(Mutex); return AllocatedAddressesMap[Address]; } + MemBufferInfo &getMemBufferInfo(ur_mem_handle_t MemBuffer) { + std::lock_guard Guard(Mutex); + return MemBufferMap[MemBuffer]; + } }; struct QueueInfo { @@ -119,10 +136,10 @@ class SanitizerInterceptor { bool updateShadowMemory(ur_queue_handle_t Queue, ur_kernel_handle_t Kernel); void enqueueAllocInfo(ur_context_handle_t Context, ur_device_handle_t Device, ur_queue_handle_t Queue, - AllocatedMemoryInfo &AllocInfo, + USMMemoryInfo &AllocInfo, ur_event_handle_t &LastEvent); bool updateHostShadowMemory(ur_context_handle_t Context, - AllocatedMemoryInfo AllocInfo); + USMMemoryInfo AllocInfo); /// Initialize Global Variables & Kernel Name at first Launch bool initKernel(ur_queue_handle_t Queue, ur_kernel_handle_t Kernel); /// Initialze USM Host Memory Pools @@ -170,3 +187,5 @@ class SanitizerInterceptor { ur_dditable_t &m_Dditable; }; + +} // namespace ur_sanitizer_layer diff --git a/source/loader/layers/sanitizer/ur_sanddi.cpp b/source/loader/layers/sanitizer/ur_sanddi.cpp index a11b854389..335074d5a3 100644 --- a/source/loader/layers/sanitizer/ur_sanddi.cpp +++ b/source/loader/layers/sanitizer/ur_sanddi.cpp @@ -10,6 +10,7 @@ * */ +#include "common.h" #include "sanitizer_interceptor.hpp" #include "ur_sanitizer_layer.hpp" @@ -18,36 +19,29 @@ namespace ur_sanitizer_layer { -// /////////////////////////////////////////////////////////////////////////////// -// /// @brief Intercept function for urUSMHostAlloc -// __urdlllocal ur_result_t UR_APICALL urUSMHostAlloc( -// ur_context_handle_t hContext, ///< [in] handle of the context object -// const ur_usm_desc_t -// *pUSMDesc, ///< [in][optional] USM memory allocation descriptor -// ur_usm_pool_handle_t -// pool, ///< [in][optional] Pointer to a pool created using urUSMPoolCreate -// size_t -// size, ///< [in] size in bytes of the USM memory object to be allocated -// void **ppMem ///< [out] pointer to USM host memory object -// ) { -// auto pfnHostAlloc = context.urDdiTable.USM.pfnHostAlloc; - -// if (nullptr == pfnHostAlloc) { -// return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -// } - -// ur_usm_host_alloc_params_t params = {&hContext, &pUSMDesc, &pool, &size, -// &ppMem}; -// uint64_t instance = context.notify_begin(UR_FUNCTION_USM_HOST_ALLOC, -// "urUSMHostAlloc", ¶ms); - -// ur_result_t result = pfnHostAlloc(hContext, pUSMDesc, pool, size, ppMem); - -// context.notify_end(UR_FUNCTION_USM_HOST_ALLOC, "urUSMHostAlloc", ¶ms, -// &result, instance); - -// return result; -// } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urUSMHostAlloc +__urdlllocal ur_result_t UR_APICALL urUSMHostAlloc( + ur_context_handle_t hContext, ///< [in] handle of the context object + const ur_usm_desc_t + *pUSMDesc, ///< [in][optional] USM memory allocation descriptor + ur_usm_pool_handle_t + pool, ///< [in][optional] Pointer to a pool created using urUSMPoolCreate + size_t + size, ///< [in] size in bytes of the USM memory object to be allocated + void **ppMem ///< [out] pointer to USM host memory object +) { + auto pfnHostAlloc = context.urDdiTable.USM.pfnHostAlloc; + + if (nullptr == pfnHostAlloc) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + + context.logger.debug("=== urUSMHostAlloc"); + + return context.interceptor->allocateMemory( + hContext, nullptr, pUSMDesc, pool, size, ppMem, USMMemoryType::HOST); +} /////////////////////////////////////////////////////////////////////////////// /// @brief Intercept function for urUSMDeviceAlloc @@ -67,68 +61,54 @@ __urdlllocal ur_result_t UR_APICALL urUSMDeviceAlloc( if (nullptr == pfnDeviceAlloc) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } - std::cerr << "=== urUSMDeviceAlloc" << std::endl; + + context.logger.debug("=== urUSMDeviceAlloc"); return context.interceptor->allocateMemory( hContext, hDevice, pUSMDesc, pool, size, ppMem, USMMemoryType::DEVICE); } -// /////////////////////////////////////////////////////////////////////////////// -// /// @brief Intercept function for urUSMSharedAlloc -// __urdlllocal ur_result_t UR_APICALL urUSMSharedAlloc( -// ur_context_handle_t hContext, ///< [in] handle of the context object -// ur_device_handle_t hDevice, ///< [in] handle of the device object -// const ur_usm_desc_t * -// pUSMDesc, ///< [in][optional] Pointer to USM memory allocation descriptor. -// ur_usm_pool_handle_t -// pool, ///< [in][optional] Pointer to a pool created using urUSMPoolCreate -// size_t -// size, ///< [in] size in bytes of the USM memory object to be allocated -// void **ppMem ///< [out] pointer to USM shared memory object -// ) { -// auto pfnSharedAlloc = context.urDdiTable.USM.pfnSharedAlloc; - -// if (nullptr == pfnSharedAlloc) { -// return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -// } - -// ur_usm_shared_alloc_params_t params = {&hContext, &hDevice, &pUSMDesc, -// &pool, &size, &ppMem}; -// uint64_t instance = context.notify_begin(UR_FUNCTION_USM_SHARED_ALLOC, -// "urUSMSharedAlloc", ¶ms); - -// ur_result_t result = -// pfnSharedAlloc(hContext, hDevice, pUSMDesc, pool, size, ppMem); - -// context.notify_end(UR_FUNCTION_USM_SHARED_ALLOC, "urUSMSharedAlloc", -// ¶ms, &result, instance); - -// return result; -// } - -// /////////////////////////////////////////////////////////////////////////////// -// /// @brief Intercept function for urUSMFree -// __urdlllocal ur_result_t UR_APICALL urUSMFree( -// ur_context_handle_t hContext, ///< [in] handle of the context object -// void *pMem ///< [in] pointer to USM memory object -// ) { -// auto pfnFree = context.urDdiTable.USM.pfnFree; - -// if (nullptr == pfnFree) { -// return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -// } - -// ur_usm_free_params_t params = {&hContext, &pMem}; -// uint64_t instance = -// context.notify_begin(UR_FUNCTION_USM_FREE, "urUSMFree", ¶ms); - -// ur_result_t result = pfnFree(hContext, pMem); - -// context.notify_end(UR_FUNCTION_USM_FREE, "urUSMFree", ¶ms, &result, -// instance); - -// return result; -// } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urUSMSharedAlloc +__urdlllocal ur_result_t UR_APICALL urUSMSharedAlloc( + ur_context_handle_t hContext, ///< [in] handle of the context object + ur_device_handle_t hDevice, ///< [in] handle of the device object + const ur_usm_desc_t * + pUSMDesc, ///< [in][optional] Pointer to USM memory allocation descriptor. + ur_usm_pool_handle_t + pool, ///< [in][optional] Pointer to a pool created using urUSMPoolCreate + size_t + size, ///< [in] size in bytes of the USM memory object to be allocated + void **ppMem ///< [out] pointer to USM shared memory object +) { + auto pfnSharedAlloc = context.urDdiTable.USM.pfnSharedAlloc; + + if (nullptr == pfnSharedAlloc) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + + context.logger.debug("=== urUSMSharedAlloc"); + + return context.interceptor->allocateMemory( + hContext, hDevice, pUSMDesc, pool, size, ppMem, USMMemoryType::SHARE); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urUSMFree +__urdlllocal ur_result_t UR_APICALL urUSMFree( + ur_context_handle_t hContext, ///< [in] handle of the context object + void *pMem ///< [in] pointer to USM memory object +) { + auto pfnFree = context.urDdiTable.USM.pfnFree; + + if (nullptr == pfnFree) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + + context.logger.debug("=== urUSMFree"); + + return context.interceptor->releaseMemory(hContext, pMem); +} /////////////////////////////////////////////////////////////////////////////// /// @brief Intercept function for urKernelCreate @@ -143,7 +123,7 @@ __urdlllocal ur_result_t UR_APICALL urKernelCreate( if (nullptr == pfnCreate) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } - std::cerr << "=== urKernelCreate" << std::endl; + // context.logger.debug("=== urKernelCreate"); ur_result_t result = pfnCreate(hProgram, pKernelName, phKernel); if (result == UR_RESULT_SUCCESS) { @@ -168,7 +148,7 @@ __urdlllocal ur_result_t UR_APICALL urQueueCreate( if (nullptr == pfnCreate) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } - std::cerr << "=== urQueueCreate" << std::endl; + // context.logger.debug("=== urQueueCreate"); ur_result_t result = pfnCreate(hContext, hDevice, pProperties, phQueue); if (result == UR_RESULT_SUCCESS) { @@ -215,7 +195,7 @@ __urdlllocal ur_result_t UR_APICALL urEnqueueKernelLaunch( return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } - std::cerr << "=== urEnqueueKernelLaunch" << std::endl; + context.logger.debug("=== urEnqueueKernelLaunch"); ur_event_handle_t lk_event{}; std::vector events(numEventsInWaitList + 1); for (unsigned i = 0; i < numEventsInWaitList; ++i) { @@ -239,6 +219,138 @@ __urdlllocal ur_result_t UR_APICALL urEnqueueKernelLaunch( return result; } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urMemBufferCreate +__urdlllocal ur_result_t UR_APICALL urMemBufferCreate( + ur_context_handle_t hContext, ///< [in] handle of the context object + ur_mem_flags_t flags, ///< [in] allocation and usage information flags + size_t size, ///< [in] size in bytes of the memory object to be allocated + const ur_buffer_properties_t + *pProperties, ///< [in][optional] pointer to buffer creation properties + ur_mem_handle_t + *phBuffer ///< [out] pointer to handle of the memory buffer created +) { + return context.interceptor->createMemoryBuffer(hContext, flags, size, + pProperties, phBuffer); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urKernelSetArgLocal +__urdlllocal ur_result_t UR_APICALL urKernelSetArgLocal( + ur_kernel_handle_t hKernel, ///< [in] handle of the kernel object + uint32_t argIndex, ///< [in] argument index in range [0, num args - 1] + size_t + argSize, ///< [in] size of the local buffer to be allocated by the runtime + const ur_kernel_arg_local_properties_t + *pProperties ///< [in][optional] pointer to local buffer properties. +) { + auto pfnSetArgLocal = context.urDdiTable.Kernel.pfnSetArgLocal; + + if (nullptr == pfnSetArgLocal) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + std::cout << "=== urKernelSetArgLocal " << argIndex << " " << argSize + << std::endl; + + uptr rzLog = ComputeRZLog(argSize); + uptr rzSize = RZLog2Size(rzLog); + uptr roundedSize = RoundUpTo(argSize, ASAN_SHADOW_GRANULARITY); + uptr neededSize = roundedSize + rzSize; + + // TODO: How to update its shadow memory? + ur_result_t result = + pfnSetArgLocal(hKernel, argIndex, neededSize, pProperties); + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urKernelSetArgMemObj +__urdlllocal ur_result_t UR_APICALL urKernelSetArgMemObj( + ur_kernel_handle_t hKernel, ///< [in] handle of the kernel object + uint32_t argIndex, ///< [in] argument index in range [0, num args - 1] + const ur_kernel_arg_mem_obj_properties_t + *pProperties, ///< [in][optional] pointer to Memory object properties. + ur_mem_handle_t hArgValue ///< [in][optional] handle of Memory object. +) { + auto pfnSetArgMemObj = context.urDdiTable.Kernel.pfnSetArgMemObj; + + if (nullptr == pfnSetArgMemObj) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + + std::cout << "=== urKernelSetArgMemObj " << argIndex << std::endl; + + ur_result_t result = + pfnSetArgMemObj(hKernel, argIndex, pProperties, hArgValue); + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Exported function for filling application's Mem table +/// with current process' addresses +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// - ::UR_RESULT_ERROR_UNSUPPORTED_VERSION +__urdlllocal ur_result_t UR_APICALL urGetMemProcAddrTable( + ur_api_version_t version, ///< [in] API version requested + ur_mem_dditable_t + *pDdiTable ///< [in,out] pointer to table of DDI function pointers +) { + // auto &dditable = ur_sanitizer_layer::context.urDdiTable.Mem; + + if (nullptr == pDdiTable) { + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + } + + if (UR_MAJOR_VERSION(ur_sanitizer_layer::context.version) != + UR_MAJOR_VERSION(version) || + UR_MINOR_VERSION(ur_sanitizer_layer::context.version) > + UR_MINOR_VERSION(version)) { + return UR_RESULT_ERROR_UNSUPPORTED_VERSION; + } + + ur_result_t result = UR_RESULT_SUCCESS; + + // dditable.pfnImageCreate = pDdiTable->pfnImageCreate; + // pDdiTable->pfnImageCreate = ur_sanitizer_layer::urMemImageCreate; + + // dditable.pfnBufferCreate = pDdiTable->pfnBufferCreate; + pDdiTable->pfnBufferCreate = ur_sanitizer_layer::urMemBufferCreate; + + // dditable.pfnRetain = pDdiTable->pfnRetain; + // pDdiTable->pfnRetain = ur_sanitizer_layer::urMemRetain; + + // dditable.pfnRelease = pDdiTable->pfnRelease; + // pDdiTable->pfnRelease = ur_sanitizer_layer::urMemRelease; + + // dditable.pfnBufferPartition = pDdiTable->pfnBufferPartition; + // pDdiTable->pfnBufferPartition = ur_sanitizer_layer::urMemBufferPartition; + + // dditable.pfnGetNativeHandle = pDdiTable->pfnGetNativeHandle; + // pDdiTable->pfnGetNativeHandle = ur_sanitizer_layer::urMemGetNativeHandle; + + // dditable.pfnBufferCreateWithNativeHandle = + // pDdiTable->pfnBufferCreateWithNativeHandle; + // pDdiTable->pfnBufferCreateWithNativeHandle = + // ur_sanitizer_layer::urMemBufferCreateWithNativeHandle; + + // dditable.pfnImageCreateWithNativeHandle = + // pDdiTable->pfnImageCreateWithNativeHandle; + // pDdiTable->pfnImageCreateWithNativeHandle = + // ur_sanitizer_layer::urMemImageCreateWithNativeHandle; + + // dditable.pfnGetInfo = pDdiTable->pfnGetInfo; + // pDdiTable->pfnGetInfo = ur_sanitizer_layer::urMemGetInfo; + + // dditable.pfnImageGetInfo = pDdiTable->pfnImageGetInfo; + // pDdiTable->pfnImageGetInfo = ur_sanitizer_layer::urMemImageGetInfo; + + return result; +} /////////////////////////////////////////////////////////////////////////////// /// @brief Exported function for filling application's Enqueue table /// with current process' addresses @@ -302,6 +414,8 @@ __urdlllocal ur_result_t UR_APICALL urGetKernelProcAddrTable( // dditable.pfnCreate = pDdiTable->pfnCreate; pDdiTable->pfnCreate = ur_sanitizer_layer::urKernelCreate; + pDdiTable->pfnSetArgLocal = ur_sanitizer_layer::urKernelSetArgLocal; + pDdiTable->pfnSetArgMemObj = ur_sanitizer_layer::urKernelSetArgMemObj; return result; } @@ -385,7 +499,7 @@ ur_result_t context_t::init(ur_dditable_t *dditable, const std::set &enabledLayerNames) { ur_result_t result = UR_RESULT_SUCCESS; - std::cout << "ur_sanitizer_layer context_t::init\n"; + context.logger.info("ur_sanitizer_layer init"); // if (!enabledLayerNames.count(name)) { // return result; @@ -406,6 +520,9 @@ ur_result_t context_t::init(ur_dditable_t *dditable, result = ur_sanitizer_layer::urGetUSMProcAddrTable( UR_API_VERSION_CURRENT, &dditable->USM); + + result = ur_sanitizer_layer::urGetMemProcAddrTable( + UR_API_VERSION_CURRENT, &dditable->Mem); } return result; diff --git a/source/loader/layers/sanitizer/ur_sanitizer_layer.cpp b/source/loader/layers/sanitizer/ur_sanitizer_layer.cpp index 6cd214e9a9..366975d574 100644 --- a/source/loader/layers/sanitizer/ur_sanitizer_layer.cpp +++ b/source/loader/layers/sanitizer/ur_sanitizer_layer.cpp @@ -20,7 +20,9 @@ namespace ur_sanitizer_layer { context_t context; /////////////////////////////////////////////////////////////////////////////// -context_t::context_t() : interceptor(new SanitizerInterceptor(urDdiTable)) {} +context_t::context_t() + : interceptor(new SanitizerInterceptor(urDdiTable)), + logger(logger::create_logger("sanitizer")) {} bool context_t::isAvailable() const { return true; } diff --git a/source/loader/layers/sanitizer/ur_sanitizer_layer.hpp b/source/loader/layers/sanitizer/ur_sanitizer_layer.hpp index fe7aafed06..843fa62a37 100644 --- a/source/loader/layers/sanitizer/ur_sanitizer_layer.hpp +++ b/source/loader/layers/sanitizer/ur_sanitizer_layer.hpp @@ -13,20 +13,23 @@ #ifndef UR_SANITIZER_LAYER_H #define UR_SANITIZER_LAYER_H 1 +#include "logger/ur_logger.hpp" #include "ur_ddi.h" #include "ur_proxy_layer.hpp" #include "ur_util.hpp" #define SANITIZER_COMP_NAME "sanitizer layer" +namespace ur_sanitizer_layer { + class SanitizerInterceptor; -namespace ur_sanitizer_layer { /////////////////////////////////////////////////////////////////////////////// class __urdlllocal context_t : public proxy_layer_context_t { public: ur_dditable_t urDdiTable = {}; SanitizerInterceptor *interceptor = {}; + logger::Logger logger; context_t(); ~context_t(); @@ -38,7 +41,7 @@ class __urdlllocal context_t : public proxy_layer_context_t { const std::set &enabledLayerNames) override; private: - const std::string name = "UR_LAYER_SANITIZER"; + const std::string name = "UR_LAYER_ASAN"; }; extern context_t context; From 25b118330387e5d8a2bc735f5e12f0026aedbabb Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Fri, 3 Nov 2023 05:58:37 -0700 Subject: [PATCH 13/72] UR Sanitizer: Add membuffer red zone marker --- .../sanitizer/sanitizer_interceptor.cpp | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/source/loader/layers/sanitizer/sanitizer_interceptor.cpp b/source/loader/layers/sanitizer/sanitizer_interceptor.cpp index 236830322e..32ec9f49f6 100644 --- a/source/loader/layers/sanitizer/sanitizer_interceptor.cpp +++ b/source/loader/layers/sanitizer/sanitizer_interceptor.cpp @@ -318,9 +318,26 @@ bool SanitizerInterceptor::updateHostShadowMemory(ur_context_handle_t Context, auto ShadowPtr = (u8 *)MemToShadow(AllocInfo.UserEnd, ShadowOffset); *ShadowPtr = Value; } - auto ShadowByte = AllocInfo.Type == USMMemoryType::DEVICE - ? kUsmDeviceRedzoneMagic - : kUsmSharedRedzoneMagic; + + u8 ShadowByte; + switch (AllocInfo.Type) { + case USMMemoryType::DEVICE: + ShadowByte = kUsmDeviceRedzoneMagic; + break; + case USMMemoryType::HOST: + ShadowByte = kUsmHostRedzoneMagic; + break; + case USMMemoryType::SHARE: + ShadowByte = kUsmSharedRedzoneMagic; + break; + case USMMemoryType::MEM_BUFFER: + ShadowByte = kMemBufferRedzoneMagic; + break; + default: + ShadowByte = 0xff; + assert(false && "Unknow AllocInfo.Type"); + } + std::memset((void *)MemToShadow(AllocInfo.AllocBegin, ShadowOffset), ShadowByte, AllocInfo.UserBegin - AllocInfo.AllocBegin); std::memset((void *)MemToShadow(tail_beg, ShadowOffset), ShadowByte, @@ -509,7 +526,7 @@ void SanitizerInterceptor::enqueueAllocInfo(ur_context_handle_t Context, assert(Res == UR_RESULT_SUCCESS); } - int ShadowByte = 0; + int ShadowByte; switch (AllocInfo.Type) { case USMMemoryType::HOST: ShadowByte = kUsmHostRedzoneMagic; @@ -520,10 +537,12 @@ void SanitizerInterceptor::enqueueAllocInfo(ur_context_handle_t Context, case USMMemoryType::SHARE: ShadowByte = kUsmSharedRedzoneMagic; break; + case USMMemoryType::MEM_BUFFER: + ShadowByte = kMemBufferRedzoneMagic; + break; default: - ShadowByte = 0xFF; + ShadowByte = 0xff; assert(false && "Unknow AllocInfo.Type"); - break; } // Left red zone @@ -726,7 +745,7 @@ ur_result_t SanitizerInterceptor::createMemoryBuffer( assert(UserEnd <= AllocEnd); auto MemoryInfo = USMMemoryInfo{AllocBegin, UserBegin, UserEnd, NeededSize, - USMMemoryType::DEVICE}; + USMMemoryType::MEM_BUFFER}; MemoryInfo.Devices.emplace(Device); // Update Shadow Memory From e6bc3060178898b0533dd58ff7b0171ff40960a0 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Mon, 6 Nov 2023 02:50:24 -0800 Subject: [PATCH 14/72] UR Sanitizer: add membuffer check --- source/loader/layers/sanitizer/device_sanitizer_report.hpp | 7 +++++-- source/loader/layers/sanitizer/ur_sanddi.cpp | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/source/loader/layers/sanitizer/device_sanitizer_report.hpp b/source/loader/layers/sanitizer/device_sanitizer_report.hpp index e65aacd5e8..b985b16748 100644 --- a/source/loader/layers/sanitizer/device_sanitizer_report.hpp +++ b/source/loader/layers/sanitizer/device_sanitizer_report.hpp @@ -13,20 +13,21 @@ namespace ur_sanitizer_layer { enum class DeviceSanitizerErrorType : int32_t { + UNKNOWN, OUT_OF_BOUND, MISALIGNED, USE_AFTER_FREE, OUT_OF_SHADOW_BOUND, - UNKNOWN }; enum class DeviceSanitizerMemoryType : int32_t { + UNKNOWN, USM_DEVICE, USM_HOST, USM_SHARED, LOCAL, PRIVATE, - UNKNOWN + MEM_BUFFER, }; // NOTE Layout of this structure should be aligned with the one in @@ -67,6 +68,8 @@ const char *DeviceSanitizerFormat(DeviceSanitizerMemoryType MemoryType) { return "Local Memory"; case DeviceSanitizerMemoryType::PRIVATE: return "Private Memory"; + case DeviceSanitizerMemoryType::MEM_BUFFER: + return "Memory Buffer"; default: return "Unknown Memory"; } diff --git a/source/loader/layers/sanitizer/ur_sanddi.cpp b/source/loader/layers/sanitizer/ur_sanddi.cpp index 335074d5a3..d89f3837ba 100644 --- a/source/loader/layers/sanitizer/ur_sanddi.cpp +++ b/source/loader/layers/sanitizer/ur_sanddi.cpp @@ -414,7 +414,7 @@ __urdlllocal ur_result_t UR_APICALL urGetKernelProcAddrTable( // dditable.pfnCreate = pDdiTable->pfnCreate; pDdiTable->pfnCreate = ur_sanitizer_layer::urKernelCreate; - pDdiTable->pfnSetArgLocal = ur_sanitizer_layer::urKernelSetArgLocal; + // pDdiTable->pfnSetArgLocal = ur_sanitizer_layer::urKernelSetArgLocal; pDdiTable->pfnSetArgMemObj = ur_sanitizer_layer::urKernelSetArgMemObj; return result; From 23fb1b36e6fd09b2fb6d00001783c18214f01a8a Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Wed, 8 Nov 2023 19:07:28 -0800 Subject: [PATCH 15/72] UR log --- source/loader/layers/sanitizer/ur_sanddi.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/loader/layers/sanitizer/ur_sanddi.cpp b/source/loader/layers/sanitizer/ur_sanddi.cpp index d89f3837ba..7cfa732680 100644 --- a/source/loader/layers/sanitizer/ur_sanddi.cpp +++ b/source/loader/layers/sanitizer/ur_sanddi.cpp @@ -249,8 +249,8 @@ __urdlllocal ur_result_t UR_APICALL urKernelSetArgLocal( if (nullptr == pfnSetArgLocal) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } - std::cout << "=== urKernelSetArgLocal " << argIndex << " " << argSize - << std::endl; + + context.logger.debug("=== urKernelSetArgLocal"); uptr rzLog = ComputeRZLog(argSize); uptr rzSize = RZLog2Size(rzLog); @@ -279,7 +279,7 @@ __urdlllocal ur_result_t UR_APICALL urKernelSetArgMemObj( return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } - std::cout << "=== urKernelSetArgMemObj " << argIndex << std::endl; + context.logger.debug("=== urKernelSetArgMemObj"); ur_result_t result = pfnSetArgMemObj(hKernel, argIndex, pProperties, hArgValue); From cb8a76106d4dad431a35df59bb62dc1ba7e998b3 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Sun, 12 Nov 2023 23:40:56 -0800 Subject: [PATCH 16/72] UR change directory --- source/loader/layers/sanitizer/{ => asan}/common.h | 0 .../layers/sanitizer/{ => asan}/device_sanitizer_report.hpp | 0 .../loader/layers/sanitizer/{ => asan}/sanitizer_interceptor.cpp | 0 .../loader/layers/sanitizer/{ => asan}/sanitizer_interceptor.hpp | 0 .../sanitizer/{ur_sanitizer_layer.cpp => asan/ur_asan_layer.cpp} | 0 .../sanitizer/{ur_sanitizer_layer.hpp => asan/ur_asan_layer.hpp} | 0 .../loader/layers/sanitizer/{ur_sanddi.cpp => asan/ur_asnddi.cpp} | 0 7 files changed, 0 insertions(+), 0 deletions(-) rename source/loader/layers/sanitizer/{ => asan}/common.h (100%) rename source/loader/layers/sanitizer/{ => asan}/device_sanitizer_report.hpp (100%) rename source/loader/layers/sanitizer/{ => asan}/sanitizer_interceptor.cpp (100%) rename source/loader/layers/sanitizer/{ => asan}/sanitizer_interceptor.hpp (100%) rename source/loader/layers/sanitizer/{ur_sanitizer_layer.cpp => asan/ur_asan_layer.cpp} (100%) rename source/loader/layers/sanitizer/{ur_sanitizer_layer.hpp => asan/ur_asan_layer.hpp} (100%) rename source/loader/layers/sanitizer/{ur_sanddi.cpp => asan/ur_asnddi.cpp} (100%) diff --git a/source/loader/layers/sanitizer/common.h b/source/loader/layers/sanitizer/asan/common.h similarity index 100% rename from source/loader/layers/sanitizer/common.h rename to source/loader/layers/sanitizer/asan/common.h diff --git a/source/loader/layers/sanitizer/device_sanitizer_report.hpp b/source/loader/layers/sanitizer/asan/device_sanitizer_report.hpp similarity index 100% rename from source/loader/layers/sanitizer/device_sanitizer_report.hpp rename to source/loader/layers/sanitizer/asan/device_sanitizer_report.hpp diff --git a/source/loader/layers/sanitizer/sanitizer_interceptor.cpp b/source/loader/layers/sanitizer/asan/sanitizer_interceptor.cpp similarity index 100% rename from source/loader/layers/sanitizer/sanitizer_interceptor.cpp rename to source/loader/layers/sanitizer/asan/sanitizer_interceptor.cpp diff --git a/source/loader/layers/sanitizer/sanitizer_interceptor.hpp b/source/loader/layers/sanitizer/asan/sanitizer_interceptor.hpp similarity index 100% rename from source/loader/layers/sanitizer/sanitizer_interceptor.hpp rename to source/loader/layers/sanitizer/asan/sanitizer_interceptor.hpp diff --git a/source/loader/layers/sanitizer/ur_sanitizer_layer.cpp b/source/loader/layers/sanitizer/asan/ur_asan_layer.cpp similarity index 100% rename from source/loader/layers/sanitizer/ur_sanitizer_layer.cpp rename to source/loader/layers/sanitizer/asan/ur_asan_layer.cpp diff --git a/source/loader/layers/sanitizer/ur_sanitizer_layer.hpp b/source/loader/layers/sanitizer/asan/ur_asan_layer.hpp similarity index 100% rename from source/loader/layers/sanitizer/ur_sanitizer_layer.hpp rename to source/loader/layers/sanitizer/asan/ur_asan_layer.hpp diff --git a/source/loader/layers/sanitizer/ur_sanddi.cpp b/source/loader/layers/sanitizer/asan/ur_asnddi.cpp similarity index 100% rename from source/loader/layers/sanitizer/ur_sanddi.cpp rename to source/loader/layers/sanitizer/asan/ur_asnddi.cpp From 5c44b1d47ca3e01fe8a9d298022296171ce51ff4 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Sun, 12 Nov 2023 23:50:05 -0800 Subject: [PATCH 17/72] UR refactor code --- source/loader/CMakeLists.txt | 16 +- source/loader/layers/sanitizer/asan/common.h | 18 +- .../asan/device_sanitizer_report.hpp | 4 +- .../sanitizer/asan/sanitizer_interceptor.cpp | 432 +++++++----------- .../sanitizer/asan/sanitizer_interceptor.hpp | 198 +++----- .../layers/sanitizer/asan/ur_asan_layer.cpp | 8 +- .../layers/sanitizer/asan/ur_asan_layer.hpp | 8 +- .../layers/sanitizer/asan/ur_asnddi.cpp | 295 +++++++++--- source/loader/ur_lib.hpp | 4 +- 9 files changed, 527 insertions(+), 456 deletions(-) diff --git a/source/loader/CMakeLists.txt b/source/loader/CMakeLists.txt index 20b93775f5..5439fff6ac 100644 --- a/source/loader/CMakeLists.txt +++ b/source/loader/CMakeLists.txt @@ -103,12 +103,16 @@ endif() if(UR_ENABLE_SANITIZER) target_sources(ur_loader PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/device_sanitizer_report.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/sanitizer_interceptor.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/sanitizer_interceptor.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/ur_sanitizer_layer.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/ur_sanitizer_layer.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/ur_sanddi.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/asan/device_sanitizer_report.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/asan/sanitizer_interceptor.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/asan/sanitizer_interceptor.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/asan/ur_asan_layer.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/asan/ur_asan_layer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/asan/ur_asnddi.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../ur/ur.cpp + ) + target_include_directories(ur_loader PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/../" ) endif() diff --git a/source/loader/layers/sanitizer/asan/common.h b/source/loader/layers/sanitizer/asan/common.h index cac9280031..968ec52c51 100644 --- a/source/loader/layers/sanitizer/asan/common.h +++ b/source/loader/layers/sanitizer/asan/common.h @@ -12,8 +12,9 @@ #include #include "ur_ddi.h" +#include -namespace ur_sanitizer_layer { +namespace ur_asan_layer { typedef uintptr_t uptr; typedef unsigned char u8; @@ -245,4 +246,17 @@ static auto getUrResultString = [](ur_result_t Result) { } }; -} // namespace ur_sanitizer_layer +// Trace an internal PI call; returns in case of an error. +#define UR_CALL(Call) \ + { \ + if (PrintTrace) \ + fprintf(stderr, "UR ---> %s\n", #Call); \ + ur_result_t Result = (Call); \ + if (PrintTrace) \ + fprintf(stderr, "UR <--- %s(%s)\n", #Call, \ + getUrResultString(Result)); \ + if (Result != UR_RESULT_SUCCESS) \ + return Result; \ + } + +} // namespace ur_asan_layer diff --git a/source/loader/layers/sanitizer/asan/device_sanitizer_report.hpp b/source/loader/layers/sanitizer/asan/device_sanitizer_report.hpp index b985b16748..9ec6a69842 100644 --- a/source/loader/layers/sanitizer/asan/device_sanitizer_report.hpp +++ b/source/loader/layers/sanitizer/asan/device_sanitizer_report.hpp @@ -10,7 +10,7 @@ #include -namespace ur_sanitizer_layer { +namespace ur_asan_layer { enum class DeviceSanitizerErrorType : int32_t { UNKNOWN, @@ -90,4 +90,4 @@ const char *DeviceSanitizerFormat(DeviceSanitizerErrorType ErrorType) { } } -} // namespace ur_sanitizer_layer +} // namespace ur_asan_layer diff --git a/source/loader/layers/sanitizer/asan/sanitizer_interceptor.cpp b/source/loader/layers/sanitizer/asan/sanitizer_interceptor.cpp index 32ec9f49f6..3d9a56bb17 100644 --- a/source/loader/layers/sanitizer/asan/sanitizer_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan/sanitizer_interceptor.cpp @@ -8,13 +8,13 @@ #include "sanitizer_interceptor.hpp" #include "device_sanitizer_report.hpp" -#include "ur_sanitizer_layer.hpp" +#include "ur_asan_layer.hpp" #include #include #include -namespace ur_sanitizer_layer { +namespace ur_asan_layer { namespace { @@ -60,6 +60,35 @@ const auto kSPIR_DeviceSanitizerReportMem = "__DeviceSanitizerReportMem"; DeviceSanitizerReport SPIR_DeviceSanitizerReportMem; +ur_context_handle_t getContext(ur_queue_handle_t Queue, + ur_dditable_t &Dditable) { + ur_context_handle_t Context; + auto Result = Dditable.Queue.pfnGetInfo(Queue, UR_QUEUE_INFO_CONTEXT, + sizeof(ur_context_handle_t), + &Context, nullptr); + assert(Result == UR_RESULT_SUCCESS); + return Context; +} + +ur_device_handle_t getDevice(ur_queue_handle_t Queue, ur_dditable_t &Dditable) { + ur_device_handle_t Device; + auto Result = + Dditable.Queue.pfnGetInfo(Queue, UR_QUEUE_INFO_DEVICE, + sizeof(ur_device_handle_t), &Device, nullptr); + assert(Result == UR_RESULT_SUCCESS); + return Device; +} + +ur_program_handle_t getProgram(ur_kernel_handle_t Kernel, + ur_dditable_t &Dditable) { + ur_program_handle_t Program; + auto Result = Dditable.Kernel.pfnGetInfo(Kernel, UR_KERNEL_INFO_PROGRAM, + sizeof(ur_program_handle_t), + &Program, nullptr); + assert(Result == UR_RESULT_SUCCESS); + return Program; +} + } // namespace ur_result_t SanitizerInterceptor::allocateMemory( @@ -70,50 +99,33 @@ ur_result_t SanitizerInterceptor::allocateMemory( assert(Alignment == 0 || IsPowerOfTwo(Alignment)); auto &ContextInfo = getContextInfo(Context); - if (!ContextInfo.Init) { - initContext(Context); - } - - if (Device) { - auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); - if (!DeviceInfo.Init) { - initDevice(Context, Device); - } - if (Alignment == 0) { - Alignment = DeviceInfo.Alignment; - } - } + auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); if (Alignment == 0) { - // FIXME: OS Defined? - Alignment = 8; + Alignment = DeviceInfo.Alignment; } - // Calcuate Size + RZSize + // Calcuate Size + Red Zone Size uptr RZLog = ComputeRZLog(Size); uptr RZSize = RZLog2Size(RZLog); uptr RoundedSize = RoundUpTo(Size, Alignment); uptr NeededSize = RoundedSize + RZSize * 2; void *Allocated = nullptr; - ur_result_t Result = UR_RESULT_SUCCESS; + if (Type == USMMemoryType::DEVICE) { - Result = m_Dditable.USM.pfnDeviceAlloc(Context, Device, Properties, - Pool, NeededSize, &Allocated); + UR_CALL(m_Dditable.USM.pfnDeviceAlloc(Context, Device, Properties, Pool, + NeededSize, &Allocated)); } else if (Type == USMMemoryType::HOST) { - Result = m_Dditable.USM.pfnHostAlloc(Context, Properties, Pool, - NeededSize, &Allocated); + UR_CALL(m_Dditable.USM.pfnHostAlloc(Context, Properties, Pool, + NeededSize, &Allocated)); } else if (Type == USMMemoryType::SHARE) { - Result = m_Dditable.USM.pfnSharedAlloc(Context, Device, Properties, - Pool, NeededSize, &Allocated); + UR_CALL(m_Dditable.USM.pfnSharedAlloc(Context, Device, Properties, Pool, + NeededSize, &Allocated)); } else { - assert(false && "SanitizerInterceptor::allocateMemory not implemented"); - } - if (Result != UR_RESULT_SUCCESS) { - return Result; + die("SanitizerInterceptor: unsupport memory type"); } - // Enqueue Shadow Memory Init uptr AllocBegin = reinterpret_cast(Allocated); uptr AllocEnd = AllocBegin + NeededSize; uptr UserBegin = AllocBegin + RZSize; @@ -126,25 +138,18 @@ ur_result_t SanitizerInterceptor::allocateMemory( *ResultPtr = reinterpret_cast(UserBegin); auto MemoryInfo = - USMMemoryInfo{AllocBegin, UserBegin, UserEnd, NeededSize, Type}; - if (Device) { - MemoryInfo.Devices.emplace(Device); - } + USMAllocInfo{AllocBegin, UserBegin, UserEnd, NeededSize, Type}; - // Update Shadow Memory - if (Device) { - auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); - std::lock_guard Guard(DeviceInfo.Mutex); + // For updating shadow memory + { + std::scoped_lock Guard(DeviceInfo.Mutex); DeviceInfo.AllocInfos.emplace_back(MemoryInfo); - } else { - std::lock_guard Guard(ContextInfo.Mutex); - ContextInfo.AllocHostInfos.emplace_back(MemoryInfo); } - // Save into AllocatedAddressesMap for releasing + // For memory release { - std::lock_guard Guard(ContextInfo.Mutex); - ContextInfo.AllocatedAddressesMap[AllocBegin] = MemoryInfo; + std::scoped_lock Guard(ContextInfo.Mutex); + ContextInfo.AllocatedUSMMap[AllocBegin] = MemoryInfo; } context.logger.info("AllocInfos:\n AllocBegin: {}\n User: {}-{}\n " @@ -157,61 +162,47 @@ ur_result_t SanitizerInterceptor::allocateMemory( ur_result_t SanitizerInterceptor::releaseMemory(ur_context_handle_t Context, void *Ptr) { auto &ContextInfo = getContextInfo(Context); - assert(ContextInfo.Init); context.logger.debug("ReleaseMemory: {}", Ptr); - std::lock_guard Guard(ContextInfo.Mutex); + std::shared_lock Guard(ContextInfo.Mutex); + auto Addr = (uptr)Ptr; // Find the last element is not greater than key - auto AddressInfoIt = - ContextInfo.AllocatedAddressesMap.upper_bound((uptr)Addr); - if (AddressInfoIt == ContextInfo.AllocatedAddressesMap.begin()) { + auto AllocInfoIt = ContextInfo.AllocatedUSMMap.upper_bound((uptr)Addr); + if (AllocInfoIt == ContextInfo.AllocatedUSMMap.begin()) { context.logger.error( "Can't find release pointer({}) in AllocatedAddressesMap", Ptr); return UR_RESULT_ERROR_INVALID_ARGUMENT; } - --AddressInfoIt; - auto &AddressInfo = AddressInfoIt->second; - std::cerr << "AddressInfo: " << AddressInfo.AllocBegin << " " - << AddressInfo.UserBegin << "\n"; - if (Addr != AddressInfo.UserBegin) { - std::cerr << "ERROR: releaseMemory failed! UserBegin\n"; - context.logger.error("Release pointer({}) is not match to {}", Ptr, - AddressInfo.UserBegin); + --AllocInfoIt; + auto &AllocInfo = AllocInfoIt->second; + + context.logger.debug("USMAllocInfo(AllocBegin={}, UserBegin={})", + AllocInfo.AllocBegin, AllocInfo.UserBegin); + + if (Addr != AllocInfo.UserBegin) { + context.logger.error("Releasing pointer({}) is not match to {}", Ptr, + AllocInfo.UserBegin); return UR_RESULT_ERROR_INVALID_ARGUMENT; } // TODO: Update shadow memory - return m_Dditable.USM.pfnFree(Context, (void *)AddressInfo.AllocBegin); -} - -void SanitizerInterceptor::addQueue(ur_context_handle_t Context, - ur_device_handle_t Device, - ur_queue_handle_t Queue) { - auto &QueueInfo = getQueueInfo(Queue); - QueueInfo.Device = Device; - QueueInfo.Context = Context; -} - -void SanitizerInterceptor::addKernel(ur_program_handle_t Program, - ur_kernel_handle_t Kernel) { - auto &KernelInfo = getKernelInfo(Kernel); - KernelInfo.Program = Program; + return m_Dditable.USM.pfnFree(Context, (void *)AllocInfo.AllocBegin); } bool SanitizerInterceptor::launchKernel(ur_kernel_handle_t Kernel, ur_queue_handle_t Queue, ur_event_handle_t &Event) { - // KernelInfo &KernelInfo = getKernelInfo(Kernel); - initKernel(Queue, Kernel); + prepareLaunch(Queue, Kernel); - updateShadowMemory(Queue, Kernel); + updateShadowMemory(Queue); - auto &QueueInfo = getQueueInfo(Queue); - std::lock_guard Guard(QueueInfo.Mutex); - Event = QueueInfo.LastEvent; - QueueInfo.LastEvent = nullptr; + auto Context = getContext(Queue, m_Dditable); + auto &ContextInfo = getContextInfo(Context); + std::lock_guard Guard(ContextInfo.LastEventMapMutex); + Event = ContextInfo.LastEventMap[Queue]; + ContextInfo.LastEventMap[Queue] = nullptr; return true; } @@ -243,19 +234,18 @@ void SanitizerInterceptor::postLaunchKernel(ur_kernel_handle_t Kernel, ur_queue_handle_t Queue, ur_event_handle_t *Event, bool SetCallback) { - auto &KernelInfo = getKernelInfo(Kernel); - auto Program = KernelInfo.Program; + auto Program = getProgram(Kernel, m_Dditable); ur_event_handle_t ReadEvent{}; // If kernel has defined SPIR_DeviceSanitizerReportMem, then we try to read it // to host, but it's okay that it isn't defined - auto Ret = m_Dditable.Enqueue.pfnDeviceGlobalVariableRead( + auto Result = m_Dditable.Enqueue.pfnDeviceGlobalVariableRead( Queue, Program, kSPIR_DeviceSanitizerReportMem, true, sizeof(SPIR_DeviceSanitizerReportMem), 0, &SPIR_DeviceSanitizerReportMem, 1, Event, &ReadEvent); - if (Ret == UR_RESULT_SUCCESS) { + if (Result == UR_RESULT_SUCCESS) { *Event = ReadEvent; auto AH = &SPIR_DeviceSanitizerReportMem; @@ -272,9 +262,8 @@ void SanitizerInterceptor::postLaunchKernel(ur_kernel_handle_t Kernel, fprintf(stderr, "%s of size %u at kernel <%s> LID(%lu, %lu, %lu) GID(%lu, " "%lu, %lu)\n", - AH->IsWrite ? "WRITE" : "READ", AH->AccessSize, - KernelInfo.Name.c_str(), AH->LID0, AH->LID1, AH->LID2, AH->GID0, - AH->GID1, AH->GID2); + AH->IsWrite ? "WRITE" : "READ", AH->AccessSize, Func, AH->LID0, + AH->LID1, AH->LID2, AH->GID0, AH->GID1, AH->GID2); fprintf(stderr, " #0 %s %s:%d\n", Func, File, AH->Line); fflush(stderr); if (!AH->IsRecover) { @@ -304,47 +293,6 @@ void SanitizerInterceptor::checkSanitizerError(ur_kernel_handle_t Kernel) { checkSanitizerReport(KernelName.c_str()); } -bool SanitizerInterceptor::updateHostShadowMemory(ur_context_handle_t Context, - USMMemoryInfo AllocInfo) { - auto &ContextInfo = getContextInfo(Context); - auto ShadowOffset = ContextInfo.HostShadowOffset; - - uptr tail_beg = RoundUpTo(AllocInfo.UserEnd, ASAN_SHADOW_GRANULARITY); - uptr tail_end = AllocInfo.AllocBegin + AllocInfo.AllocSize; - // user tail - if (tail_beg != AllocInfo.UserEnd) { - auto Value = AllocInfo.UserEnd - - RoundDownTo(AllocInfo.UserEnd, ASAN_SHADOW_GRANULARITY); - auto ShadowPtr = (u8 *)MemToShadow(AllocInfo.UserEnd, ShadowOffset); - *ShadowPtr = Value; - } - - u8 ShadowByte; - switch (AllocInfo.Type) { - case USMMemoryType::DEVICE: - ShadowByte = kUsmDeviceRedzoneMagic; - break; - case USMMemoryType::HOST: - ShadowByte = kUsmHostRedzoneMagic; - break; - case USMMemoryType::SHARE: - ShadowByte = kUsmSharedRedzoneMagic; - break; - case USMMemoryType::MEM_BUFFER: - ShadowByte = kMemBufferRedzoneMagic; - break; - default: - ShadowByte = 0xff; - assert(false && "Unknow AllocInfo.Type"); - } - - std::memset((void *)MemToShadow(AllocInfo.AllocBegin, ShadowOffset), - ShadowByte, AllocInfo.UserBegin - AllocInfo.AllocBegin); - std::memset((void *)MemToShadow(tail_beg, ShadowOffset), ShadowByte, - tail_end - tail_beg); - return true; -} - uptr MemToShadow_CPU(uptr USM_SHADOW_BASE, uptr UPtr) { return USM_SHADOW_BASE + (UPtr >> 3); } @@ -366,11 +314,8 @@ uptr MemToShadow_DG2(uptr USM_SHADOW_BASE, uptr UPtr) { } } -ur_result_t -SanitizerInterceptor::piextMemAllocShadow(ur_context_handle_t Context, - ur_device_handle_t Device) { - auto &ContextInfo = getContextInfo(Context); - auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); +ur_result_t SanitizerInterceptor::allocShadowMemory(ur_context_handle_t Context, + DeviceInfo &DeviceInfo) { if (DeviceInfo.Type == UR_DEVICE_TYPE_CPU) { DeviceInfo.ShadowOffset = 0x00007fff7fffULL; DeviceInfo.ShadowOffsetEnd = 0x10007fff7fffULL; @@ -382,18 +327,16 @@ SanitizerInterceptor::piextMemAllocShadow(ur_context_handle_t Context, constexpr size_t SHADOW_SIZE = 1ULL << 46; // TODO: Protect Bad Zone - auto URes = m_Dditable.VirtualMem.pfnReserve( - Context, nullptr, SHADOW_SIZE, (void **)&DeviceInfo.ShadowOffset); - if (URes != UR_RESULT_SUCCESS) { - context.logger.error("urVirtualMemReserve: {}", - getUrResultString(URes)); - return URes; - } + UR_CALL(m_Dditable.VirtualMem.pfnReserve( + Context, nullptr, SHADOW_SIZE, (void **)&DeviceInfo.ShadowOffset)); DeviceInfo.ShadowOffsetEnd = DeviceInfo.ShadowOffset + SHADOW_SIZE; } else { - assert(false && "Unsupport device type"); + die("Unsupport device type"); } + context.logger.info("Device ShadowOffset: {} - {}", + (void *)DeviceInfo.ShadowOffset, + (void *)DeviceInfo.ShadowOffsetEnd); return UR_RESULT_SUCCESS; } @@ -405,7 +348,7 @@ ur_result_t SanitizerInterceptor::piextEnqueueMemSetShadow( auto &ContextInfo = getContextInfo(Context); auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); if (DeviceInfo.Type == UR_DEVICE_TYPE_CPU) { - assert(false && "Unsupport device type"); + die("Unsupport device type"); } else if (DeviceInfo.Type == UR_DEVICE_TYPE_GPU) { const uptr UPtr = (uptr)Ptr; @@ -487,7 +430,7 @@ ur_result_t SanitizerInterceptor::piextEnqueueMemSetShadow( return URes; } } else { - assert(false && "Unsupport device type"); + die("Unsupport device type"); } return UR_RESULT_SUCCESS; } @@ -502,16 +445,13 @@ ur_result_t SanitizerInterceptor::enqueuePoisonShadow( Value, NumEvents, EventsList, OutEvent); } -void SanitizerInterceptor::enqueueAllocInfo(ur_context_handle_t Context, - ur_device_handle_t Device, - ur_queue_handle_t Queue, - USMMemoryInfo &AllocInfo, - ur_event_handle_t &LastEvent) { +ur_result_t SanitizerInterceptor::enqueueAllocInfo( + ur_context_handle_t Context, ur_device_handle_t Device, + ur_queue_handle_t Queue, USMAllocInfo &AllocInfo, + ur_event_handle_t &LastEvent) { // Init zero - auto Res = - enqueuePoisonShadow(Context, Device, Queue, AllocInfo.AllocBegin, - AllocInfo.AllocSize, 0, LastEvent, &LastEvent); - assert(Res == UR_RESULT_SUCCESS); + UR_CALL(enqueuePoisonShadow(Context, Device, Queue, AllocInfo.AllocBegin, + AllocInfo.AllocSize, 0, LastEvent, &LastEvent)); uptr TailBegin = RoundUpTo(AllocInfo.UserEnd, ASAN_SHADOW_GRANULARITY); uptr TailEnd = AllocInfo.AllocBegin + AllocInfo.AllocSize; @@ -520,10 +460,8 @@ void SanitizerInterceptor::enqueueAllocInfo(ur_context_handle_t Context, if (TailBegin != AllocInfo.UserEnd) { auto Value = AllocInfo.UserEnd - RoundDownTo(AllocInfo.UserEnd, ASAN_SHADOW_GRANULARITY); - auto Res = - enqueuePoisonShadow(Context, Device, Queue, AllocInfo.UserEnd, 1, - Value, LastEvent, &LastEvent); - assert(Res == UR_RESULT_SUCCESS); + UR_CALL(enqueuePoisonShadow(Context, Device, Queue, AllocInfo.UserEnd, + 1, Value, LastEvent, &LastEvent)); } int ShadowByte; @@ -546,130 +484,109 @@ void SanitizerInterceptor::enqueueAllocInfo(ur_context_handle_t Context, } // Left red zone - Res = enqueuePoisonShadow(Context, Device, Queue, AllocInfo.AllocBegin, - AllocInfo.UserBegin - AllocInfo.AllocBegin, - ShadowByte, LastEvent, &LastEvent); - assert(Res == UR_RESULT_SUCCESS); + UR_CALL(enqueuePoisonShadow(Context, Device, Queue, AllocInfo.AllocBegin, + AllocInfo.UserBegin - AllocInfo.AllocBegin, + ShadowByte, LastEvent, &LastEvent)); // Right red zone - Res = enqueuePoisonShadow(Context, Device, Queue, TailBegin, - TailEnd - TailBegin, ShadowByte, LastEvent, - &LastEvent); - assert(Res == UR_RESULT_SUCCESS); + UR_CALL(enqueuePoisonShadow(Context, Device, Queue, TailBegin, + TailEnd - TailBegin, ShadowByte, LastEvent, + &LastEvent)); + + return UR_RESULT_SUCCESS; } -bool SanitizerInterceptor::updateShadowMemory(ur_queue_handle_t Queue, - ur_kernel_handle_t Kernel) { - (void)Kernel; - auto &QueueInfo = getQueueInfo(Queue); - auto Context = QueueInfo.Context; - auto Device = QueueInfo.Device; +ur_result_t SanitizerInterceptor::updateShadowMemory(ur_queue_handle_t Queue) { + auto Context = getContext(Queue, m_Dditable); + auto Device = getDevice(Queue, m_Dditable); + assert(Device != nullptr); auto &ContextInfo = getContextInfo(Context); - assert(ContextInfo.Init); + + auto &HostInfo = ContextInfo.getDeviceInfo(nullptr); auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); - assert(DeviceInfo.Init); - std::lock_guard QueueGuard(QueueInfo.Mutex); - std::lock_guard DeviceGuard(DeviceInfo.Mutex); + std::shared_lock HostGuard(HostInfo.Mutex, + std::defer_lock); + std::unique_lock DeviceGuard(DeviceInfo.Mutex, + std::defer_lock); + std::scoped_lock, + std::unique_lock, ur_mutex> + Guard(HostGuard, DeviceGuard, ContextInfo.LastEventMapMutex); - ur_event_handle_t LastEvent = QueueInfo.LastEvent; + ur_event_handle_t LastEvent = ContextInfo.LastEventMap[Queue]; // FIXME: Always update host USM, but it'd be better to update host USM // selectively, or each devices once - for (auto &AllocInfo : ContextInfo.AllocHostInfos) { - if (AllocInfo.Devices.find(Device) == AllocInfo.Devices.end()) { - enqueueAllocInfo(Context, Device, Queue, AllocInfo, LastEvent); - AllocInfo.Devices.emplace(Device); - } + for (auto &AllocInfo : HostInfo.AllocInfos) { + UR_CALL(enqueueAllocInfo(Context, Device, Queue, AllocInfo, LastEvent)); } for (auto &AllocInfo : DeviceInfo.AllocInfos) { - enqueueAllocInfo(Context, Device, Queue, AllocInfo, LastEvent); + UR_CALL(enqueueAllocInfo(Context, Device, Queue, AllocInfo, LastEvent)); } DeviceInfo.AllocInfos.clear(); - QueueInfo.LastEvent = LastEvent; + ContextInfo.LastEventMap[Queue] = LastEvent; - return true; + return UR_RESULT_SUCCESS; } -void SanitizerInterceptor::initContext(ur_context_handle_t Context) { - auto &ContextInfo = getContextInfo(Context); - std::lock_guard Guard(ContextInfo.Mutex); +ur_result_t SanitizerInterceptor::addContext(ur_context_handle_t Context) { + auto ContextInfoPtr = std::make_unique(); - if (ContextInfo.Init) { - return; - } + // Host Device + auto DeviceInfoPtr = std::make_unique(); + DeviceInfoPtr->Type = UR_DEVICE_TYPE_CPU; + DeviceInfoPtr->Alignment = ASAN_SHADOW_GRANULARITY; - ContextInfo.Init = true; -} + // TODO: Check if host asan is enabled + DeviceInfoPtr->ShadowOffset = 0; + DeviceInfoPtr->ShadowOffsetEnd = 0; -void SanitizerInterceptor::initDevice(ur_context_handle_t Context, - ur_device_handle_t Device) { - auto &ContextInfo = getContextInfo(Context); - assert(ContextInfo.Init && "Context not inited"); + ContextInfoPtr->DeviceMap.emplace(nullptr, std::move(DeviceInfoPtr)); - assert(Device); - auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); - std::lock_guard Guard(DeviceInfo.Mutex); + std::scoped_lock Guard(m_ContextMapMutex); + m_ContextMap.emplace(Context, std::move(ContextInfoPtr)); - if (DeviceInfo.Init) { - return; - } + return UR_RESULT_SUCCESS; +} - // Query Device Type - auto Result = m_Dditable.Device.pfnGetInfo(Device, UR_DEVICE_INFO_TYPE, - sizeof(DeviceInfo.Type), - &DeviceInfo.Type, nullptr); - assert(Result == UR_RESULT_SUCCESS); +ur_result_t SanitizerInterceptor::addDevice(ur_context_handle_t Context, + ur_device_handle_t Device) { + auto DeviceInfoPtr = std::make_unique(); + + // Query device type + UR_CALL(m_Dditable.Device.pfnGetInfo(Device, UR_DEVICE_INFO_TYPE, + sizeof(DeviceInfoPtr->Type), + &DeviceInfoPtr->Type, nullptr)); // Query alignment - Result = m_Dditable.Device.pfnGetInfo( + UR_CALL(m_Dditable.Device.pfnGetInfo( Device, UR_DEVICE_INFO_MEM_BASE_ADDR_ALIGN, - sizeof(DeviceInfo.Alignment), &DeviceInfo.Alignment, nullptr); - assert(Result == UR_RESULT_SUCCESS); + sizeof(DeviceInfoPtr->Alignment), &DeviceInfoPtr->Alignment, nullptr)); // Allocate shadow memory - Result = piextMemAllocShadow(Context, Device); - assert(Result == UR_RESULT_SUCCESS); + UR_CALL(allocShadowMemory(Context, *DeviceInfoPtr.get())); - context.logger.info("Device ShadowOffset: {} - {}", - (void *)DeviceInfo.ShadowOffset, - (void *)DeviceInfo.ShadowOffsetEnd); + auto &ContextInfo = getContextInfo(Context); + std::scoped_lock Guard(ContextInfo.Mutex); + ContextInfo.DeviceMap.emplace(Device, std::move(DeviceInfoPtr)); - DeviceInfo.Init = true; + return UR_RESULT_SUCCESS; } -bool SanitizerInterceptor::initKernel(ur_queue_handle_t Queue, - ur_kernel_handle_t Kernel) { - auto &KernelInfo = getKernelInfo(Kernel); - auto Program = KernelInfo.Program; - - auto &QueueInfo = getQueueInfo(Queue); - auto Device = QueueInfo.Device; - auto Context = QueueInfo.Context; +void SanitizerInterceptor::prepareLaunch(ur_queue_handle_t Queue, + ur_kernel_handle_t Kernel) { + auto Context = getContext(Queue, m_Dditable); + auto Device = getDevice(Queue, m_Dditable); + auto Program = getProgram(Kernel, m_Dditable); auto &ContextInfo = getContextInfo(Context); - if (!ContextInfo.Init) { - initContext(Context); - } - auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); - if (!DeviceInfo.Init) { - initDevice(Context, Device); - } - // Get kernel name - { - std::lock_guard KernelGuard(KernelInfo.Mutex); - if (KernelInfo.Name.empty()) { - KernelInfo.Name = getKernelName(Kernel); - } - } - - std::lock_guard QueueGuard(QueueInfo.Mutex); - ur_event_handle_t LastEvent = QueueInfo.LastEvent; + std::scoped_lock Guard(ContextInfo.LastEventMapMutex); + ur_event_handle_t LastEvent = ContextInfo.LastEventMap[Queue]; do { // Set global variable to program @@ -697,20 +614,15 @@ bool SanitizerInterceptor::initKernel(ur_queue_handle_t Queue, &DeviceInfo.ShadowOffsetEnd); } while (false); - QueueInfo.LastEvent = LastEvent; - - return true; + ContextInfo.LastEventMap[Queue] = LastEvent; } ur_result_t SanitizerInterceptor::createMemoryBuffer( ur_context_handle_t Context, ur_mem_flags_t Flags, size_t Size, const ur_buffer_properties_t *Properties, ur_mem_handle_t *Buffer) { auto &ContextInfo = getContextInfo(Context); - if (!ContextInfo.Init) { - initContext(Context); - } - constexpr size_t Alignment = 8; + constexpr size_t Alignment = ASAN_SHADOW_GRANULARITY; // Calcuate Size + RZSize const uptr RZLog = ComputeRZLog(Size); @@ -733,7 +645,9 @@ ur_result_t SanitizerInterceptor::createMemoryBuffer( void *Allocated = (void *)NativeHandle; ur_device_handle_t Device; - Result = m_Dditable.USM.pfnGetMemAllocInfo(Context, Allocated, UR_USM_ALLOC_INFO_DEVICE, sizeof(Device), &Device, nullptr); + Result = m_Dditable.USM.pfnGetMemAllocInfo( + Context, Allocated, UR_USM_ALLOC_INFO_DEVICE, + sizeof(ur_device_handle_t), &Device, nullptr); uptr AllocBegin = reinterpret_cast(Allocated); uptr AllocEnd = AllocBegin + NeededSize; @@ -744,23 +658,23 @@ ur_result_t SanitizerInterceptor::createMemoryBuffer( uptr UserEnd = UserBegin + Size; assert(UserEnd <= AllocEnd); - auto MemoryInfo = USMMemoryInfo{AllocBegin, UserBegin, UserEnd, NeededSize, - USMMemoryType::MEM_BUFFER}; - MemoryInfo.Devices.emplace(Device); + auto MemoryInfo = USMAllocInfo{AllocBegin, UserBegin, UserEnd, NeededSize, + USMMemoryType::MEM_BUFFER}; - // Update Shadow Memory - auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); - std::lock_guard Guard(DeviceInfo.Mutex); - DeviceInfo.AllocInfos.emplace_back(MemoryInfo); + { + auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); + std::scoped_lock Guard(DeviceInfo.Mutex); + DeviceInfo.AllocInfos.emplace_back(MemoryInfo); + } - // Save into AllocatedAddressesMap for releasing { - std::lock_guard Guard(ContextInfo.Mutex); - ContextInfo.AllocatedAddressesMap[AllocBegin] = MemoryInfo; + std::scoped_lock Guard(ContextInfo.Mutex); + ContextInfo.AllocatedUSMMap[AllocBegin] = MemoryInfo; } // Create a subbuffer which is surrounded by red zone - ur_buffer_region_t BufferRegion{UR_STRUCTURE_TYPE_BUFFER_REGION, nullptr, UserBegin - AllocBegin, Size}; + ur_buffer_region_t BufferRegion{UR_STRUCTURE_TYPE_BUFFER_REGION, nullptr, + UserBegin - AllocBegin, Size}; ur_mem_handle_t SubBuffer; Result = m_Dditable.Mem.pfnBufferPartition(*Buffer, Flags, UR_BUFFER_CREATE_TYPE_REGION, @@ -773,4 +687,4 @@ ur_result_t SanitizerInterceptor::createMemoryBuffer( return UR_RESULT_SUCCESS; } -} // namespace ur_sanitizer_layer +} // namespace ur_asan_layer diff --git a/source/loader/layers/sanitizer/asan/sanitizer_interceptor.hpp b/source/loader/layers/sanitizer/asan/sanitizer_interceptor.hpp index 97b0c794d0..0dd920bf2b 100644 --- a/source/loader/layers/sanitizer/asan/sanitizer_interceptor.hpp +++ b/source/loader/layers/sanitizer/asan/sanitizer_interceptor.hpp @@ -14,19 +14,61 @@ #include #include #include -#include #include #include #include #include -namespace ur_sanitizer_layer { +namespace ur_asan_layer { -enum USMMemoryType { - DEVICE, - SHARE, - HOST, - MEM_BUFFER +enum USMMemoryType { DEVICE, SHARE, HOST, MEM_BUFFER }; + +struct USMAllocInfo { + uptr AllocBegin; + uptr UserBegin; + uptr UserEnd; + size_t AllocSize; + USMMemoryType Type; +}; + +enum class DeviceType { CPU, GPU_PVC, GPU_DG2 }; + +struct DeviceInfo { + ur_device_type_t Type; + size_t Alignment; + uptr ShadowOffset; + uptr ShadowOffsetEnd; + + // Lock InitPool & AllocInfos + ur_shared_mutex Mutex; + std::vector AllocInfos; +}; + +struct ContextInfo { + + DeviceInfo &getDeviceInfo(ur_device_handle_t Device) { + std::shared_lock Guard(Mutex); + assert(DeviceMap.find(Device) != DeviceMap.end()); + return *DeviceMap[Device].get(); + } + + USMAllocInfo &getUSMAllocInfo(uptr Address) { + std::shared_lock Guard(Mutex); + assert(AllocatedUSMMap.find(Address) != AllocatedUSMMap.end()); + return AllocatedUSMMap[Address]; + } + + ur_mutex LastEventMapMutex; + std::unordered_map LastEventMap; + + ur_shared_mutex Mutex; + // Note: nullptr is host device + std::unordered_map> DeviceMap; + + /// key: USMAllocInfo.AllocBegin + /// value: USMAllocInfo + /// Use AllocBegin as key can help to detect underflow pointer + std::map AllocatedUSMMap; }; class SanitizerInterceptor { @@ -39,117 +81,36 @@ class SanitizerInterceptor { ur_usm_pool_handle_t Pool, size_t Size, void **ResultPtr, USMMemoryType Type); ur_result_t releaseMemory(ur_context_handle_t Context, void *Ptr); - void addQueue(ur_context_handle_t Context, ur_device_handle_t Device, - ur_queue_handle_t Queue); - void addKernel(ur_program_handle_t Program, ur_kernel_handle_t Kernel); + bool launchKernel(ur_kernel_handle_t Kernel, ur_queue_handle_t Queue, ur_event_handle_t &Event); void postLaunchKernel(ur_kernel_handle_t Kernel, ur_queue_handle_t Queue, ur_event_handle_t *Event, bool SetCallback = true); + void checkSanitizerError(ur_kernel_handle_t Kernel); ur_result_t createMemoryBuffer(ur_context_handle_t Context, ur_mem_flags_t Flags, size_t Size, const ur_buffer_properties_t *Properties, ur_mem_handle_t *Buffer); - private: - struct USMMemoryInfo { - uptr AllocBegin; - uptr UserBegin; - uptr UserEnd; - size_t AllocSize; - USMMemoryType Type; - std::unordered_set - Devices; // host USM have a lot of devices - }; - - struct DeviceInfo { - bool Init = false; - bool InitPool = false; - - ur_device_type_t Type; - size_t Alignment; - uptr ShadowOffset; - uptr ShadowOffsetEnd; - - std::vector AllocInfos; - std::mutex Mutex; // Lock Init & InitPool & AllocInfos - }; - - struct MemBufferInfo { - ur_mem_handle_t Buffer; - USMMemoryInfo AllocInfo; - size_t Size; - size_t RZSize; - }; - - enum class DeviceType { CPU, GPU_PVC, GPU_DG2 }; - - struct ContextInfo { - bool Init = false; - - /// USMMemoryInfo.AllocBegin => USMMemoryInfo - /// - /// Use AllocBegin as key can help to detect underflow pointer - std::map AllocatedAddressesMap; - - std::vector AllocHostInfos; - - /// Each context is able to contain multiple devices - std::unordered_map DeviceMap; - - /// - std::unordered_map MemBufferMap; - - std::mutex Mutex; // Lock Init and Maps - - uptr HostShadowOffset; - - DeviceInfo &getDeviceInfo(ur_device_handle_t Device) { - std::lock_guard Guard(Mutex); - return DeviceMap[Device]; - } - USMMemoryInfo &getUSMMemoryInfo(uptr Address) { - std::lock_guard Guard(Mutex); - return AllocatedAddressesMap[Address]; - } - MemBufferInfo &getMemBufferInfo(ur_mem_handle_t MemBuffer) { - std::lock_guard Guard(Mutex); - return MemBufferMap[MemBuffer]; - } - }; - - struct QueueInfo { - ur_device_handle_t Device; - ur_context_handle_t Context; - ur_event_handle_t LastEvent = nullptr; - std::mutex Mutex; // Lock LastEvent - }; - - struct KernelInfo { - ur_program_handle_t Program = nullptr; - std::string Name; - std::mutex Mutex; // Lock Name - }; + ur_result_t addContext(ur_context_handle_t Context); + ur_result_t addDevice(ur_context_handle_t Context, + ur_device_handle_t Device); private: - bool updateShadowMemory(ur_queue_handle_t Queue, ur_kernel_handle_t Kernel); - void enqueueAllocInfo(ur_context_handle_t Context, - ur_device_handle_t Device, ur_queue_handle_t Queue, - USMMemoryInfo &AllocInfo, - ur_event_handle_t &LastEvent); - bool updateHostShadowMemory(ur_context_handle_t Context, - USMMemoryInfo AllocInfo); + ur_result_t updateShadowMemory(ur_queue_handle_t Queue); + ur_result_t enqueueAllocInfo(ur_context_handle_t Context, + ur_device_handle_t Device, + ur_queue_handle_t Queue, + USMAllocInfo &AlloccInfo, + ur_event_handle_t &LastEvent); + /// Initialize Global Variables & Kernel Name at first Launch - bool initKernel(ur_queue_handle_t Queue, ur_kernel_handle_t Kernel); - /// Initialze USM Host Memory Pools - void initContext(ur_context_handle_t Context); - /// Initialze USM Device & Shared Memory Pools, Privte & Local Memory Shadow - /// Pools - void initDevice(ur_context_handle_t Context, ur_device_handle_t Device); + void prepareLaunch(ur_queue_handle_t Queue, ur_kernel_handle_t Kernel); + std::string getKernelName(ur_kernel_handle_t Kernel); - ur_result_t piextMemAllocShadow(ur_context_handle_t Context, - ur_device_handle_t Device); + ur_result_t allocShadowMemory(ur_context_handle_t Context, + DeviceInfo &DeviceInfo); ur_result_t piextEnqueueMemSetShadow(ur_context_handle_t Context, ur_device_handle_t Device, ur_queue_handle_t Queue, void *Addr, @@ -163,29 +124,18 @@ class SanitizerInterceptor { uptr Size, u8 Value, ur_event_handle_t DepEvent, ur_event_handle_t *OutEvent); - - private: - std::unordered_map m_ContextMap; - std::mutex m_ContextMapMutex; - std::unordered_map m_QueueMap; - std::mutex m_QueueMapMutex; - std::unordered_map m_KernelMap; - std::mutex m_KernelMapMutex; - ContextInfo &getContextInfo(ur_context_handle_t Context) { - std::lock_guard ContextMapGuard(m_ContextMapMutex); - return m_ContextMap[Context]; - } - QueueInfo &getQueueInfo(ur_queue_handle_t Queue) { - std::lock_guard QueueMapGuard(m_QueueMapMutex); - return m_QueueMap[Queue]; - } - KernelInfo &getKernelInfo(ur_kernel_handle_t Kernel) { - std::lock_guard KernelMapGuard(m_KernelMapMutex); - return m_KernelMap[Kernel]; + std::shared_lock Guard(m_ContextMapMutex); + assert(m_ContextMap.find(Context) != m_ContextMap.end()); + return *m_ContextMap[Context].get(); } + private: + std::unordered_map> + m_ContextMap; + ur_shared_mutex m_ContextMapMutex; + ur_dditable_t &m_Dditable; }; -} // namespace ur_sanitizer_layer +} // namespace ur_asan_layer diff --git a/source/loader/layers/sanitizer/asan/ur_asan_layer.cpp b/source/loader/layers/sanitizer/asan/ur_asan_layer.cpp index 366975d574..b352ddbf6a 100644 --- a/source/loader/layers/sanitizer/asan/ur_asan_layer.cpp +++ b/source/loader/layers/sanitizer/asan/ur_asan_layer.cpp @@ -6,17 +6,17 @@ * See LICENSE.TXT * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception * - * @file ur_sanitizer_layer.cpp + * @file ur_asan_layer.cpp * */ -#include "ur_sanitizer_layer.hpp" +#include "ur_asan_layer.hpp" #include "sanitizer_interceptor.hpp" #include "ur_api.h" #include "ur_util.hpp" #include -namespace ur_sanitizer_layer { +namespace ur_asan_layer { context_t context; /////////////////////////////////////////////////////////////////////////////// @@ -28,4 +28,4 @@ bool context_t::isAvailable() const { return true; } /////////////////////////////////////////////////////////////////////////////// context_t::~context_t() {} -} // namespace ur_sanitizer_layer +} // namespace ur_asan_layer diff --git a/source/loader/layers/sanitizer/asan/ur_asan_layer.hpp b/source/loader/layers/sanitizer/asan/ur_asan_layer.hpp index 843fa62a37..d92eb5839a 100644 --- a/source/loader/layers/sanitizer/asan/ur_asan_layer.hpp +++ b/source/loader/layers/sanitizer/asan/ur_asan_layer.hpp @@ -6,7 +6,7 @@ * See LICENSE.TXT * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception * - * @file ur_sanitizer_layer.h + * @file ur_asan_layer.h * */ @@ -18,9 +18,9 @@ #include "ur_proxy_layer.hpp" #include "ur_util.hpp" -#define SANITIZER_COMP_NAME "sanitizer layer" +#define SANITIZER_COMP_NAME "address sanitizer layer" -namespace ur_sanitizer_layer { +namespace ur_asan_layer { class SanitizerInterceptor; @@ -45,6 +45,6 @@ class __urdlllocal context_t : public proxy_layer_context_t { }; extern context_t context; -} // namespace ur_sanitizer_layer +} // namespace ur_asan_layer #endif /* UR_sanitizer_LAYER_H */ diff --git a/source/loader/layers/sanitizer/asan/ur_asnddi.cpp b/source/loader/layers/sanitizer/asan/ur_asnddi.cpp index 7cfa732680..b17fb0565d 100644 --- a/source/loader/layers/sanitizer/asan/ur_asnddi.cpp +++ b/source/loader/layers/sanitizer/asan/ur_asnddi.cpp @@ -12,12 +12,12 @@ #include "common.h" #include "sanitizer_interceptor.hpp" -#include "ur_sanitizer_layer.hpp" +#include "ur_asan_layer.hpp" #include #include -namespace ur_sanitizer_layer { +namespace ur_asan_layer { /////////////////////////////////////////////////////////////////////////////// /// @brief Intercept function for urUSMHostAlloc @@ -126,9 +126,9 @@ __urdlllocal ur_result_t UR_APICALL urKernelCreate( // context.logger.debug("=== urKernelCreate"); ur_result_t result = pfnCreate(hProgram, pKernelName, phKernel); - if (result == UR_RESULT_SUCCESS) { - context.interceptor->addKernel(hProgram, *phKernel); - } + // if (result == UR_RESULT_SUCCESS) { + // context.interceptor->addKernel(hProgram, *phKernel); + // } return result; } @@ -151,9 +151,9 @@ __urdlllocal ur_result_t UR_APICALL urQueueCreate( // context.logger.debug("=== urQueueCreate"); ur_result_t result = pfnCreate(hContext, hDevice, pProperties, phQueue); - if (result == UR_RESULT_SUCCESS) { - context.interceptor->addQueue(hContext, hDevice, *phQueue); - } + // if (result == UR_RESULT_SUCCESS) { + // context.interceptor->addQueue(hContext, hDevice, *phQueue); + // } return result; } @@ -287,6 +287,136 @@ __urdlllocal ur_result_t UR_APICALL urKernelSetArgMemObj( return result; } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urContextCreate +__urdlllocal ur_result_t UR_APICALL urContextCreate( + uint32_t numDevices, ///< [in] the number of devices given in phDevices + const ur_device_handle_t + *phDevices, ///< [in][range(0, numDevices)] array of handle of devices. + const ur_context_properties_t * + pProperties, ///< [in][optional] pointer to context creation properties. + ur_context_handle_t + *phContext ///< [out] pointer to handle of context object created +) { + auto pfnCreate = context.urDdiTable.Context.pfnCreate; + + if (nullptr == pfnCreate) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + + ur_result_t result = + pfnCreate(numDevices, phDevices, pProperties, phContext); + + if (result == UR_RESULT_SUCCESS) { + auto Context = *phContext; + result = context.interceptor->addContext(Context); + if (result != UR_RESULT_SUCCESS) { + return result; + } + for (uint32_t i = 0; i < numDevices; ++i) { + result = context.interceptor->addDevice(Context, phDevices[i]); + if (result != UR_RESULT_SUCCESS) { + return result; + } + } + } + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urContextCreateWithNativeHandle +__urdlllocal ur_result_t UR_APICALL urContextCreateWithNativeHandle( + ur_native_handle_t + hNativeContext, ///< [in][nocheck] the native handle of the context. + uint32_t numDevices, ///< [in] number of devices associated with the context + const ur_device_handle_t * + phDevices, ///< [in][range(0, numDevices)] list of devices associated with the context + const ur_context_native_properties_t * + pProperties, ///< [in][optional] pointer to native context properties struct + ur_context_handle_t * + phContext ///< [out] pointer to the handle of the context object created. +) { + auto pfnCreateWithNativeHandle = + context.urDdiTable.Context.pfnCreateWithNativeHandle; + + if (nullptr == pfnCreateWithNativeHandle) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + + ur_result_t result = pfnCreateWithNativeHandle( + hNativeContext, numDevices, phDevices, pProperties, phContext); + + if (result == UR_RESULT_SUCCESS) { + auto Context = *phContext; + result = context.interceptor->addContext(Context); + if (result != UR_RESULT_SUCCESS) { + return result; + } + for (uint32_t i = 0; i < numDevices; ++i) { + result = context.interceptor->addDevice(Context, phDevices[i]); + if (result != UR_RESULT_SUCCESS) { + return result; + } + } + } + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Exported function for filling application's Context table +/// with current process' addresses +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// - ::UR_RESULT_ERROR_UNSUPPORTED_VERSION +__urdlllocal ur_result_t UR_APICALL urGetContextProcAddrTable( + ur_api_version_t version, ///< [in] API version requested + ur_context_dditable_t + *pDdiTable ///< [in,out] pointer to table of DDI function pointers +) { + auto &dditable = ur_asan_layer::context.urDdiTable.Context; + + if (nullptr == pDdiTable) { + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + } + + if (UR_MAJOR_VERSION(ur_asan_layer::context.version) != + UR_MAJOR_VERSION(version) || + UR_MINOR_VERSION(ur_asan_layer::context.version) > + UR_MINOR_VERSION(version)) { + return UR_RESULT_ERROR_UNSUPPORTED_VERSION; + } + + ur_result_t result = UR_RESULT_SUCCESS; + + dditable.pfnCreate = pDdiTable->pfnCreate; + pDdiTable->pfnCreate = ur_asan_layer::urContextCreate; + + // dditable.pfnRetain = pDdiTable->pfnRetain; + // pDdiTable->pfnRetain = ur_asan_layer::urContextRetain; + + // dditable.pfnRelease = pDdiTable->pfnRelease; + // pDdiTable->pfnRelease = ur_asan_layer::urContextRelease; + + // dditable.pfnGetInfo = pDdiTable->pfnGetInfo; + // pDdiTable->pfnGetInfo = ur_asan_layer::urContextGetInfo; + + // dditable.pfnGetNativeHandle = pDdiTable->pfnGetNativeHandle; + // pDdiTable->pfnGetNativeHandle = ur_asan_layer::urContextGetNativeHandle; + + dditable.pfnCreateWithNativeHandle = pDdiTable->pfnCreateWithNativeHandle; + pDdiTable->pfnCreateWithNativeHandle = + ur_asan_layer::urContextCreateWithNativeHandle; + + // dditable.pfnSetExtendedDeleter = pDdiTable->pfnSetExtendedDeleter; + // pDdiTable->pfnSetExtendedDeleter = + // ur_asan_layer::urContextSetExtendedDeleter; + + return result; +} /////////////////////////////////////////////////////////////////////////////// /// @brief Exported function for filling application's Mem table /// with current process' addresses @@ -300,15 +430,15 @@ __urdlllocal ur_result_t UR_APICALL urGetMemProcAddrTable( ur_mem_dditable_t *pDdiTable ///< [in,out] pointer to table of DDI function pointers ) { - // auto &dditable = ur_sanitizer_layer::context.urDdiTable.Mem; + // auto &dditable = ur_asan_layer::context.urDdiTable.Mem; if (nullptr == pDdiTable) { return UR_RESULT_ERROR_INVALID_NULL_POINTER; } - if (UR_MAJOR_VERSION(ur_sanitizer_layer::context.version) != + if (UR_MAJOR_VERSION(ur_asan_layer::context.version) != UR_MAJOR_VERSION(version) || - UR_MINOR_VERSION(ur_sanitizer_layer::context.version) > + UR_MINOR_VERSION(ur_asan_layer::context.version) > UR_MINOR_VERSION(version)) { return UR_RESULT_ERROR_UNSUPPORTED_VERSION; } @@ -316,38 +446,38 @@ __urdlllocal ur_result_t UR_APICALL urGetMemProcAddrTable( ur_result_t result = UR_RESULT_SUCCESS; // dditable.pfnImageCreate = pDdiTable->pfnImageCreate; - // pDdiTable->pfnImageCreate = ur_sanitizer_layer::urMemImageCreate; + // pDdiTable->pfnImageCreate = ur_asan_layer::urMemImageCreate; // dditable.pfnBufferCreate = pDdiTable->pfnBufferCreate; - pDdiTable->pfnBufferCreate = ur_sanitizer_layer::urMemBufferCreate; + pDdiTable->pfnBufferCreate = ur_asan_layer::urMemBufferCreate; // dditable.pfnRetain = pDdiTable->pfnRetain; - // pDdiTable->pfnRetain = ur_sanitizer_layer::urMemRetain; + // pDdiTable->pfnRetain = ur_asan_layer::urMemRetain; // dditable.pfnRelease = pDdiTable->pfnRelease; - // pDdiTable->pfnRelease = ur_sanitizer_layer::urMemRelease; + // pDdiTable->pfnRelease = ur_asan_layer::urMemRelease; // dditable.pfnBufferPartition = pDdiTable->pfnBufferPartition; - // pDdiTable->pfnBufferPartition = ur_sanitizer_layer::urMemBufferPartition; + // pDdiTable->pfnBufferPartition = ur_asan_layer::urMemBufferPartition; // dditable.pfnGetNativeHandle = pDdiTable->pfnGetNativeHandle; - // pDdiTable->pfnGetNativeHandle = ur_sanitizer_layer::urMemGetNativeHandle; + // pDdiTable->pfnGetNativeHandle = ur_asan_layer::urMemGetNativeHandle; // dditable.pfnBufferCreateWithNativeHandle = // pDdiTable->pfnBufferCreateWithNativeHandle; // pDdiTable->pfnBufferCreateWithNativeHandle = - // ur_sanitizer_layer::urMemBufferCreateWithNativeHandle; + // ur_asan_layer::urMemBufferCreateWithNativeHandle; // dditable.pfnImageCreateWithNativeHandle = // pDdiTable->pfnImageCreateWithNativeHandle; // pDdiTable->pfnImageCreateWithNativeHandle = - // ur_sanitizer_layer::urMemImageCreateWithNativeHandle; + // ur_asan_layer::urMemImageCreateWithNativeHandle; // dditable.pfnGetInfo = pDdiTable->pfnGetInfo; - // pDdiTable->pfnGetInfo = ur_sanitizer_layer::urMemGetInfo; + // pDdiTable->pfnGetInfo = ur_asan_layer::urMemGetInfo; // dditable.pfnImageGetInfo = pDdiTable->pfnImageGetInfo; - // pDdiTable->pfnImageGetInfo = ur_sanitizer_layer::urMemImageGetInfo; + // pDdiTable->pfnImageGetInfo = ur_asan_layer::urMemImageGetInfo; return result; } @@ -364,15 +494,15 @@ __urdlllocal ur_result_t UR_APICALL urGetEnqueueProcAddrTable( ur_enqueue_dditable_t *pDdiTable ///< [in,out] pointer to table of DDI function pointers ) { - // auto &dditable = ur_sanitizer_layer::context.urDdiTable.Enqueue; + // auto &dditable = ur_asan_layer::context.urDdiTable.Enqueue; if (nullptr == pDdiTable) { return UR_RESULT_ERROR_INVALID_NULL_POINTER; } - if (UR_MAJOR_VERSION(ur_sanitizer_layer::context.version) != + if (UR_MAJOR_VERSION(ur_asan_layer::context.version) != UR_MAJOR_VERSION(version) || - UR_MINOR_VERSION(ur_sanitizer_layer::context.version) > + UR_MINOR_VERSION(ur_asan_layer::context.version) > UR_MINOR_VERSION(version)) { return UR_RESULT_ERROR_UNSUPPORTED_VERSION; } @@ -380,7 +510,7 @@ __urdlllocal ur_result_t UR_APICALL urGetEnqueueProcAddrTable( ur_result_t result = UR_RESULT_SUCCESS; // dditable.pfnKernelLaunch = pDdiTable->pfnKernelLaunch; - pDdiTable->pfnKernelLaunch = ur_sanitizer_layer::urEnqueueKernelLaunch; + pDdiTable->pfnKernelLaunch = ur_asan_layer::urEnqueueKernelLaunch; return result; } @@ -397,15 +527,15 @@ __urdlllocal ur_result_t UR_APICALL urGetKernelProcAddrTable( ur_kernel_dditable_t *pDdiTable ///< [in,out] pointer to table of DDI function pointers ) { - // auto &dditable = ur_sanitizer_layer::context.urDdiTable.Kernel; + // auto &dditable = ur_asan_layer::context.urDdiTable.Kernel; if (nullptr == pDdiTable) { return UR_RESULT_ERROR_INVALID_NULL_POINTER; } - if (UR_MAJOR_VERSION(ur_sanitizer_layer::context.version) != + if (UR_MAJOR_VERSION(ur_asan_layer::context.version) != UR_MAJOR_VERSION(version) || - UR_MINOR_VERSION(ur_sanitizer_layer::context.version) > + UR_MINOR_VERSION(ur_asan_layer::context.version) > UR_MINOR_VERSION(version)) { return UR_RESULT_ERROR_UNSUPPORTED_VERSION; } @@ -413,9 +543,9 @@ __urdlllocal ur_result_t UR_APICALL urGetKernelProcAddrTable( ur_result_t result = UR_RESULT_SUCCESS; // dditable.pfnCreate = pDdiTable->pfnCreate; - pDdiTable->pfnCreate = ur_sanitizer_layer::urKernelCreate; - // pDdiTable->pfnSetArgLocal = ur_sanitizer_layer::urKernelSetArgLocal; - pDdiTable->pfnSetArgMemObj = ur_sanitizer_layer::urKernelSetArgMemObj; + pDdiTable->pfnCreate = ur_asan_layer::urKernelCreate; + // pDdiTable->pfnSetArgLocal = ur_asan_layer::urKernelSetArgLocal; + pDdiTable->pfnSetArgMemObj = ur_asan_layer::urKernelSetArgMemObj; return result; } @@ -432,15 +562,15 @@ __urdlllocal ur_result_t UR_APICALL urGetQueueProcAddrTable( ur_queue_dditable_t *pDdiTable ///< [in,out] pointer to table of DDI function pointers ) { - // auto &dditable = ur_sanitizer_layer::context.urDdiTable.Queue; + // auto &dditable = ur_asan_layer::context.urDdiTable.Queue; if (nullptr == pDdiTable) { return UR_RESULT_ERROR_INVALID_NULL_POINTER; } - if (UR_MAJOR_VERSION(ur_sanitizer_layer::context.version) != + if (UR_MAJOR_VERSION(ur_asan_layer::context.version) != UR_MAJOR_VERSION(version) || - UR_MINOR_VERSION(ur_sanitizer_layer::context.version) > + UR_MINOR_VERSION(ur_asan_layer::context.version) > UR_MINOR_VERSION(version)) { return UR_RESULT_ERROR_UNSUPPORTED_VERSION; } @@ -448,7 +578,7 @@ __urdlllocal ur_result_t UR_APICALL urGetQueueProcAddrTable( ur_result_t result = UR_RESULT_SUCCESS; // dditable.pfnCreate = pDdiTable->pfnCreate; - pDdiTable->pfnCreate = ur_sanitizer_layer::urQueueCreate; + pDdiTable->pfnCreate = ur_asan_layer::urQueueCreate; return result; } @@ -465,15 +595,15 @@ __urdlllocal ur_result_t UR_APICALL urGetUSMProcAddrTable( ur_usm_dditable_t *pDdiTable ///< [in,out] pointer to table of DDI function pointers ) { - // auto &dditable = ur_sanitizer_layer::context.urDdiTable.USM; + // auto &dditable = ur_asan_layer::context.urDdiTable.USM; if (nullptr == pDdiTable) { return UR_RESULT_ERROR_INVALID_NULL_POINTER; } - if (UR_MAJOR_VERSION(ur_sanitizer_layer::context.version) != + if (UR_MAJOR_VERSION(ur_asan_layer::context.version) != UR_MAJOR_VERSION(version) || - UR_MINOR_VERSION(ur_sanitizer_layer::context.version) > + UR_MINOR_VERSION(ur_asan_layer::context.version) > UR_MINOR_VERSION(version)) { return UR_RESULT_ERROR_UNSUPPORTED_VERSION; } @@ -481,16 +611,75 @@ __urdlllocal ur_result_t UR_APICALL urGetUSMProcAddrTable( ur_result_t result = UR_RESULT_SUCCESS; // dditable.pfnHostAlloc = pDdiTable->pfnHostAlloc; - // pDdiTable->pfnHostAlloc = ur_sanitizer_layer::urUSMHostAlloc; + // pDdiTable->pfnHostAlloc = ur_asan_layer::urUSMHostAlloc; // dditable.pfnDeviceAlloc = pDdiTable->pfnDeviceAlloc; - pDdiTable->pfnDeviceAlloc = ur_sanitizer_layer::urUSMDeviceAlloc; + pDdiTable->pfnDeviceAlloc = ur_asan_layer::urUSMDeviceAlloc; // dditable.pfnSharedAlloc = pDdiTable->pfnSharedAlloc; - // pDdiTable->pfnSharedAlloc = ur_sanitizer_layer::urUSMSharedAlloc; + // pDdiTable->pfnSharedAlloc = ur_asan_layer::urUSMSharedAlloc; // dditable.pfnFree = pDdiTable->pfnFree; - // pDdiTable->pfnFree = ur_sanitizer_layer::urUSMFree; + // pDdiTable->pfnFree = ur_asan_layer::urUSMFree; + + return result; +} +/////////////////////////////////////////////////////////////////////////////// +/// @brief Exported function for filling application's Device table +/// with current process' addresses +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// - ::UR_RESULT_ERROR_UNSUPPORTED_VERSION +__urdlllocal ur_result_t UR_APICALL urGetDeviceProcAddrTable( + ur_api_version_t version, ///< [in] API version requested + ur_device_dditable_t + *pDdiTable ///< [in,out] pointer to table of DDI function pointers +) { + // auto &dditable = ur_asan_layer::context.urDdiTable.Device; + + if (nullptr == pDdiTable) { + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + } + + if (UR_MAJOR_VERSION(ur_asan_layer::context.version) != + UR_MAJOR_VERSION(version) || + UR_MINOR_VERSION(ur_asan_layer::context.version) > + UR_MINOR_VERSION(version)) { + return UR_RESULT_ERROR_UNSUPPORTED_VERSION; + } + + ur_result_t result = UR_RESULT_SUCCESS; + + // dditable.pfnGet = pDdiTable->pfnGet; + // pDdiTable->pfnGet = ur_asan_layer::urDeviceGet; + + // dditable.pfnGetInfo = pDdiTable->pfnGetInfo; + // pDdiTable->pfnGetInfo = ur_asan_layer::urDeviceGetInfo; + + // dditable.pfnRetain = pDdiTable->pfnRetain; + // pDdiTable->pfnRetain = ur_asan_layer::urDeviceRetain; + + // dditable.pfnRelease = pDdiTable->pfnRelease; + // pDdiTable->pfnRelease = ur_asan_layer::urDeviceRelease; + + // dditable.pfnPartition = pDdiTable->pfnPartition; + // pDdiTable->pfnPartition = ur_asan_layer::urDevicePartition; + + // dditable.pfnSelectBinary = pDdiTable->pfnSelectBinary; + // pDdiTable->pfnSelectBinary = ur_asan_layer::urDeviceSelectBinary; + + // dditable.pfnGetNativeHandle = pDdiTable->pfnGetNativeHandle; + // pDdiTable->pfnGetNativeHandle = ur_asan_layer::urDeviceGetNativeHandle; + + // dditable.pfnCreateWithNativeHandle = pDdiTable->pfnCreateWithNativeHandle; + // pDdiTable->pfnCreateWithNativeHandle = + // ur_asan_layer::urDeviceCreateWithNativeHandle; + + // dditable.pfnGetGlobalTimestamps = pDdiTable->pfnGetGlobalTimestamps; + // pDdiTable->pfnGetGlobalTimestamps = + // ur_asan_layer::urDeviceGetGlobalTimestamps; return result; } @@ -499,7 +688,7 @@ ur_result_t context_t::init(ur_dditable_t *dditable, const std::set &enabledLayerNames) { ur_result_t result = UR_RESULT_SUCCESS; - context.logger.info("ur_sanitizer_layer init"); + context.logger.info("ur_asan_layer init"); // if (!enabledLayerNames.count(name)) { // return result; @@ -509,22 +698,22 @@ ur_result_t context_t::init(ur_dditable_t *dditable, // FIXME: Just copy needed APIs? urDdiTable = *dditable; - result = ur_sanitizer_layer::urGetEnqueueProcAddrTable( + result = ur_asan_layer::urGetEnqueueProcAddrTable( UR_API_VERSION_CURRENT, &dditable->Enqueue); - result = ur_sanitizer_layer::urGetKernelProcAddrTable( - UR_API_VERSION_CURRENT, &dditable->Kernel); + result = ur_asan_layer::urGetKernelProcAddrTable(UR_API_VERSION_CURRENT, + &dditable->Kernel); - result = ur_sanitizer_layer::urGetQueueProcAddrTable( - UR_API_VERSION_CURRENT, &dditable->Queue); + result = ur_asan_layer::urGetQueueProcAddrTable(UR_API_VERSION_CURRENT, + &dditable->Queue); - result = ur_sanitizer_layer::urGetUSMProcAddrTable( - UR_API_VERSION_CURRENT, &dditable->USM); + result = ur_asan_layer::urGetUSMProcAddrTable(UR_API_VERSION_CURRENT, + &dditable->USM); - result = ur_sanitizer_layer::urGetMemProcAddrTable( - UR_API_VERSION_CURRENT, &dditable->Mem); + result = ur_asan_layer::urGetMemProcAddrTable(UR_API_VERSION_CURRENT, + &dditable->Mem); } return result; } -} // namespace ur_sanitizer_layer +} // namespace ur_asan_layer diff --git a/source/loader/ur_lib.hpp b/source/loader/ur_lib.hpp index 12f3877932..f754d214f2 100644 --- a/source/loader/ur_lib.hpp +++ b/source/loader/ur_lib.hpp @@ -23,7 +23,7 @@ #include "tracing/ur_tracing_layer.hpp" #endif #if UR_ENABLE_SANITIZER -#include "sanitizer/ur_sanitizer_layer.hpp" +#include "sanitizer/asan/ur_asan_layer.hpp" #endif #include @@ -72,7 +72,7 @@ class __urdlllocal context_t { &ur_tracing_layer::context, #endif #if UR_ENABLE_SANITIZER - &ur_sanitizer_layer::context + &ur_asan_layer::context #endif }; std::string availableLayers; From a377c945181f2ef3b83eab6d3f63ab8087decc41 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Mon, 13 Nov 2023 02:06:23 -0800 Subject: [PATCH 18/72] UR refactor code --- .../sanitizer/asan/sanitizer_interceptor.cpp | 36 ++++++++++++++----- .../sanitizer/asan/sanitizer_interceptor.hpp | 20 ++++++++--- .../layers/sanitizer/asan/ur_asnddi.cpp | 12 +++++-- 3 files changed, 52 insertions(+), 16 deletions(-) diff --git a/source/loader/layers/sanitizer/asan/sanitizer_interceptor.cpp b/source/loader/layers/sanitizer/asan/sanitizer_interceptor.cpp index 3d9a56bb17..ea69dee4a9 100644 --- a/source/loader/layers/sanitizer/asan/sanitizer_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan/sanitizer_interceptor.cpp @@ -198,11 +198,15 @@ bool SanitizerInterceptor::launchKernel(ur_kernel_handle_t Kernel, updateShadowMemory(Queue); + // Return LastEvent in QueueInfo auto Context = getContext(Queue, m_Dditable); auto &ContextInfo = getContextInfo(Context); - std::lock_guard Guard(ContextInfo.LastEventMapMutex); - Event = ContextInfo.LastEventMap[Queue]; - ContextInfo.LastEventMap[Queue] = nullptr; + auto &QueueInfo = ContextInfo.getQueueInfo(Queue); + + std::scoped_lock Guard(QueueInfo.Mutex); + Event = QueueInfo.LastEvent; + QueueInfo.LastEvent = nullptr; + return true; } @@ -505,6 +509,7 @@ ur_result_t SanitizerInterceptor::updateShadowMemory(ur_queue_handle_t Queue) { auto &HostInfo = ContextInfo.getDeviceInfo(nullptr); auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); + auto &QueueInfo = ContextInfo.getQueueInfo(Queue); std::shared_lock HostGuard(HostInfo.Mutex, std::defer_lock); @@ -512,9 +517,9 @@ ur_result_t SanitizerInterceptor::updateShadowMemory(ur_queue_handle_t Queue) { std::defer_lock); std::scoped_lock, std::unique_lock, ur_mutex> - Guard(HostGuard, DeviceGuard, ContextInfo.LastEventMapMutex); + Guard(HostGuard, DeviceGuard, QueueInfo.Mutex); - ur_event_handle_t LastEvent = ContextInfo.LastEventMap[Queue]; + ur_event_handle_t LastEvent = QueueInfo.LastEvent; // FIXME: Always update host USM, but it'd be better to update host USM // selectively, or each devices once @@ -527,7 +532,7 @@ ur_result_t SanitizerInterceptor::updateShadowMemory(ur_queue_handle_t Queue) { } DeviceInfo.AllocInfos.clear(); - ContextInfo.LastEventMap[Queue] = LastEvent; + QueueInfo.LastEvent = LastEvent; return UR_RESULT_SUCCESS; } @@ -576,6 +581,18 @@ ur_result_t SanitizerInterceptor::addDevice(ur_context_handle_t Context, return UR_RESULT_SUCCESS; } +ur_result_t SanitizerInterceptor::addQueue(ur_context_handle_t Context, + ur_queue_handle_t Queue) { + auto QueueInfoPtr = std::make_unique(); + QueueInfoPtr->LastEvent = nullptr; + + auto &ContextInfo = getContextInfo(Context); + std::scoped_lock Guard(ContextInfo.Mutex); + ContextInfo.QueueMap.emplace(Queue, std::move(QueueInfoPtr)); + + return UR_RESULT_SUCCESS; +} + void SanitizerInterceptor::prepareLaunch(ur_queue_handle_t Queue, ur_kernel_handle_t Kernel) { auto Context = getContext(Queue, m_Dditable); @@ -584,9 +601,10 @@ void SanitizerInterceptor::prepareLaunch(ur_queue_handle_t Queue, auto &ContextInfo = getContextInfo(Context); auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); + auto &QueueInfo = ContextInfo.getQueueInfo(Queue); - std::scoped_lock Guard(ContextInfo.LastEventMapMutex); - ur_event_handle_t LastEvent = ContextInfo.LastEventMap[Queue]; + std::scoped_lock Guard(QueueInfo.Mutex); + ur_event_handle_t LastEvent = QueueInfo.LastEvent; do { // Set global variable to program @@ -614,7 +632,7 @@ void SanitizerInterceptor::prepareLaunch(ur_queue_handle_t Queue, &DeviceInfo.ShadowOffsetEnd); } while (false); - ContextInfo.LastEventMap[Queue] = LastEvent; + QueueInfo.LastEvent = LastEvent; } ur_result_t SanitizerInterceptor::createMemoryBuffer( diff --git a/source/loader/layers/sanitizer/asan/sanitizer_interceptor.hpp b/source/loader/layers/sanitizer/asan/sanitizer_interceptor.hpp index 0dd920bf2b..b7a54a0c9b 100644 --- a/source/loader/layers/sanitizer/asan/sanitizer_interceptor.hpp +++ b/source/loader/layers/sanitizer/asan/sanitizer_interceptor.hpp @@ -44,6 +44,11 @@ struct DeviceInfo { std::vector AllocInfos; }; +struct QueueInfo { + ur_mutex Mutex; + ur_event_handle_t LastEvent; +}; + struct ContextInfo { DeviceInfo &getDeviceInfo(ur_device_handle_t Device) { @@ -52,18 +57,23 @@ struct ContextInfo { return *DeviceMap[Device].get(); } + QueueInfo &getQueueInfo(ur_queue_handle_t Queue) { + std::shared_lock Guard(Mutex); + assert(QueueMap.find(Queue) != QueueMap.end()); + return *QueueMap[Queue].get(); + } + USMAllocInfo &getUSMAllocInfo(uptr Address) { std::shared_lock Guard(Mutex); assert(AllocatedUSMMap.find(Address) != AllocatedUSMMap.end()); return AllocatedUSMMap[Address]; } - ur_mutex LastEventMapMutex; - std::unordered_map LastEventMap; - ur_shared_mutex Mutex; // Note: nullptr is host device - std::unordered_map> DeviceMap; + std::unordered_map> + DeviceMap; + std::unordered_map> QueueMap; /// key: USMAllocInfo.AllocBegin /// value: USMAllocInfo @@ -96,6 +106,7 @@ class SanitizerInterceptor { ur_result_t addContext(ur_context_handle_t Context); ur_result_t addDevice(ur_context_handle_t Context, ur_device_handle_t Device); + ur_result_t addQueue(ur_context_handle_t Context, ur_queue_handle_t Queue); private: ur_result_t updateShadowMemory(ur_queue_handle_t Queue); @@ -124,6 +135,7 @@ class SanitizerInterceptor { uptr Size, u8 Value, ur_event_handle_t DepEvent, ur_event_handle_t *OutEvent); + ContextInfo &getContextInfo(ur_context_handle_t Context) { std::shared_lock Guard(m_ContextMapMutex); assert(m_ContextMap.find(Context) != m_ContextMap.end()); diff --git a/source/loader/layers/sanitizer/asan/ur_asnddi.cpp b/source/loader/layers/sanitizer/asan/ur_asnddi.cpp index b17fb0565d..4018a4ea11 100644 --- a/source/loader/layers/sanitizer/asan/ur_asnddi.cpp +++ b/source/loader/layers/sanitizer/asan/ur_asnddi.cpp @@ -151,9 +151,9 @@ __urdlllocal ur_result_t UR_APICALL urQueueCreate( // context.logger.debug("=== urQueueCreate"); ur_result_t result = pfnCreate(hContext, hDevice, pProperties, phQueue); - // if (result == UR_RESULT_SUCCESS) { - // context.interceptor->addQueue(hContext, hDevice, *phQueue); - // } + if (result == UR_RESULT_SUCCESS) { + result = context.interceptor->addQueue(hContext, *phQueue); + } return result; } @@ -698,6 +698,12 @@ ur_result_t context_t::init(ur_dditable_t *dditable, // FIXME: Just copy needed APIs? urDdiTable = *dditable; + result = ur_asan_layer::urGetContextProcAddrTable( + UR_API_VERSION_CURRENT, &dditable->Context); + + // result = ur_asan_layer::urGetDeviceProcAddrTable( + // UR_API_VERSION_CURRENT, &dditable->Device); + result = ur_asan_layer::urGetEnqueueProcAddrTable( UR_API_VERSION_CURRENT, &dditable->Enqueue); From 96eea7aff8f5facea7fdfc46870b5f64f26a4a26 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Mon, 13 Nov 2023 22:17:58 -0800 Subject: [PATCH 19/72] UR: WIP partition membuffer --- .../sanitizer/asan/sanitizer_interceptor.cpp | 118 ++++++++---------- .../sanitizer/asan/sanitizer_interceptor.hpp | 12 +- .../layers/sanitizer/asan/ur_asnddi.cpp | 24 +++- 3 files changed, 86 insertions(+), 68 deletions(-) diff --git a/source/loader/layers/sanitizer/asan/sanitizer_interceptor.cpp b/source/loader/layers/sanitizer/asan/sanitizer_interceptor.cpp index ea69dee4a9..32103f292f 100644 --- a/source/loader/layers/sanitizer/asan/sanitizer_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan/sanitizer_interceptor.cpp @@ -60,6 +60,27 @@ const auto kSPIR_DeviceSanitizerReportMem = "__DeviceSanitizerReportMem"; DeviceSanitizerReport SPIR_DeviceSanitizerReportMem; +uptr MemToShadow_CPU(uptr USM_SHADOW_BASE, uptr UPtr) { + return USM_SHADOW_BASE + (UPtr >> 3); +} + +uptr MemToShadow_PVC(uptr USM_SHADOW_BASE, uptr UPtr) { + if (UPtr & 0xFF00000000000000ULL) { // Device USM + return USM_SHADOW_BASE + 0x200000000000ULL + + ((UPtr & 0xFFFFFFFFFFFFULL) >> 3); + } else { // Only consider 47bit VA + return USM_SHADOW_BASE + ((UPtr & 0x7FFFFFFFFFFFULL) >> 3); + } +} + +uptr MemToShadow_DG2(uptr USM_SHADOW_BASE, uptr UPtr) { + if (UPtr & (~0xFFFFFFFFFFFFULL)) { // Device USM + return USM_SHADOW_BASE + ((UPtr & 0xFFFFFFFFFFFFULL) >> 3); + } else { + return USM_SHADOW_BASE + (UPtr >> 3); + } +} + ur_context_handle_t getContext(ur_queue_handle_t Queue, ur_dditable_t &Dditable) { ur_context_handle_t Context; @@ -210,30 +231,6 @@ bool SanitizerInterceptor::launchKernel(ur_kernel_handle_t Kernel, return true; } -static void checkSanitizerReport(const char *KernelName) { - auto AH = &SPIR_DeviceSanitizerReportMem; - if (!AH->Flag) { - return; - } - - const char *File = AH->File[0] ? AH->File : ""; - const char *Func = AH->Func[0] ? AH->Func : ""; - - fprintf(stderr, "\n====ERROR: DeviceSanitizer: %s on %s\n\n", - DeviceSanitizerFormat(AH->ErrorType), - DeviceSanitizerFormat(AH->MemoryType)); - fprintf(stderr, - "%s of size %u at kernel <%s> LID(%lu, %lu, %lu) GID(%lu, " - "%lu, %lu)\n", - AH->IsWrite ? "WRITE" : "READ", AH->AccessSize, KernelName, - AH->LID0, AH->LID1, AH->LID2, AH->GID0, AH->GID1, AH->GID2); - fprintf(stderr, " #0 %s %s:%d\n", Func, File, AH->Line); - fflush(stderr); - if (!AH->IsRecover) { - abort(); - } -} - void SanitizerInterceptor::postLaunchKernel(ur_kernel_handle_t Kernel, ur_queue_handle_t Queue, ur_event_handle_t *Event, @@ -292,38 +289,12 @@ std::string SanitizerInterceptor::getKernelName(ur_kernel_handle_t Kernel) { return std::string(KernelNameBuf.data(), KernelNameSize); } -void SanitizerInterceptor::checkSanitizerError(ur_kernel_handle_t Kernel) { - std::string KernelName = getKernelName(Kernel); - checkSanitizerReport(KernelName.c_str()); -} - -uptr MemToShadow_CPU(uptr USM_SHADOW_BASE, uptr UPtr) { - return USM_SHADOW_BASE + (UPtr >> 3); -} - -uptr MemToShadow_PVC(uptr USM_SHADOW_BASE, uptr UPtr) { - if (UPtr & 0xFF00000000000000ULL) { // Device USM - return USM_SHADOW_BASE + 0x200000000000ULL + - ((UPtr & 0xFFFFFFFFFFFFULL) >> 3); - } else { // Only consider 47bit VA - return USM_SHADOW_BASE + ((UPtr & 0x7FFFFFFFFFFFULL) >> 3); - } -} - -uptr MemToShadow_DG2(uptr USM_SHADOW_BASE, uptr UPtr) { - if (UPtr & (~0xFFFFFFFFFFFFULL)) { // Device USM - return USM_SHADOW_BASE + ((UPtr & 0xFFFFFFFFFFFFULL) >> 3); - } else { - return USM_SHADOW_BASE + (UPtr >> 3); - } -} - ur_result_t SanitizerInterceptor::allocShadowMemory(ur_context_handle_t Context, DeviceInfo &DeviceInfo) { - if (DeviceInfo.Type == UR_DEVICE_TYPE_CPU) { + if (DeviceInfo.Type == DeviceType::CPU) { DeviceInfo.ShadowOffset = 0x00007fff7fffULL; DeviceInfo.ShadowOffsetEnd = 0x10007fff7fffULL; - } else if (DeviceInfo.Type == UR_DEVICE_TYPE_GPU) { + } else if (DeviceInfo.Type == DeviceType::GPU_PVC) { /// SHADOW MEMORY MAPPING (PVC, with CPU 47bit) /// Host/Shared USM : 0x0 ~ 0x0fff_ffff_ffff /// ? : 0x1000_0000_0000 ~ 0x1fff_ffff_ffff @@ -351,9 +322,9 @@ ur_result_t SanitizerInterceptor::piextEnqueueMemSetShadow( ur_event_handle_t *OutEvent) { auto &ContextInfo = getContextInfo(Context); auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); - if (DeviceInfo.Type == UR_DEVICE_TYPE_CPU) { + if (DeviceInfo.Type == DeviceType::CPU) { die("Unsupport device type"); - } else if (DeviceInfo.Type == UR_DEVICE_TYPE_GPU) { + } else if (DeviceInfo.Type == DeviceType::GPU_PVC) { const uptr UPtr = (uptr)Ptr; ur_event_handle_t InternalEvent{}; @@ -542,7 +513,7 @@ ur_result_t SanitizerInterceptor::addContext(ur_context_handle_t Context) { // Host Device auto DeviceInfoPtr = std::make_unique(); - DeviceInfoPtr->Type = UR_DEVICE_TYPE_CPU; + DeviceInfoPtr->Type = DeviceType::CPU; DeviceInfoPtr->Alignment = ASAN_SHADOW_GRANULARITY; // TODO: Check if host asan is enabled @@ -562,9 +533,19 @@ ur_result_t SanitizerInterceptor::addDevice(ur_context_handle_t Context, auto DeviceInfoPtr = std::make_unique(); // Query device type - UR_CALL(m_Dditable.Device.pfnGetInfo(Device, UR_DEVICE_INFO_TYPE, - sizeof(DeviceInfoPtr->Type), - &DeviceInfoPtr->Type, nullptr)); + ur_device_type_t DeviceType; + UR_CALL(m_Dditable.Device.pfnGetInfo( + Device, UR_DEVICE_INFO_TYPE, sizeof(DeviceType), &DeviceType, nullptr)); + switch (DeviceType) { + case UR_DEVICE_TYPE_CPU: + DeviceInfoPtr->Type = DeviceType::CPU; + break; + case UR_DEVICE_TYPE_GPU: + DeviceInfoPtr->Type = DeviceType::GPU_PVC; + break; + default: + DeviceInfoPtr->Type = DeviceType::UNKNOWN; + } // Query alignment UR_CALL(m_Dditable.Device.pfnGetInfo( @@ -637,7 +618,7 @@ void SanitizerInterceptor::prepareLaunch(ur_queue_handle_t Queue, ur_result_t SanitizerInterceptor::createMemoryBuffer( ur_context_handle_t Context, ur_mem_flags_t Flags, size_t Size, - const ur_buffer_properties_t *Properties, ur_mem_handle_t *Buffer) { + const ur_buffer_properties_t *Properties, ur_mem_handle_t *OutBuffer) { auto &ContextInfo = getContextInfo(Context); constexpr size_t Alignment = ASAN_SHADOW_GRANULARITY; @@ -649,14 +630,14 @@ ur_result_t SanitizerInterceptor::createMemoryBuffer( const uptr NeededSize = RoundedSize + RZSize * 2; auto Result = m_Dditable.Mem.pfnBufferCreate(Context, Flags, NeededSize, - Properties, Buffer); + Properties, OutBuffer); if (Result != UR_RESULT_SUCCESS) { return Result; } // Get Native Handle ur_native_handle_t NativeHandle{}; - Result = m_Dditable.Mem.pfnGetNativeHandle(*Buffer, &NativeHandle); + Result = m_Dditable.Mem.pfnGetNativeHandle(*OutBuffer, &NativeHandle); if (Result != UR_RESULT_SUCCESS) { return Result; } @@ -694,15 +675,26 @@ ur_result_t SanitizerInterceptor::createMemoryBuffer( ur_buffer_region_t BufferRegion{UR_STRUCTURE_TYPE_BUFFER_REGION, nullptr, UserBegin - AllocBegin, Size}; ur_mem_handle_t SubBuffer; - Result = m_Dditable.Mem.pfnBufferPartition(*Buffer, Flags, + Result = m_Dditable.Mem.pfnBufferPartition(*OutBuffer, Flags, UR_BUFFER_CREATE_TYPE_REGION, &BufferRegion, &SubBuffer); if (Result != UR_RESULT_SUCCESS) { return Result; } - *Buffer = SubBuffer; + *OutBuffer = SubBuffer; return UR_RESULT_SUCCESS; } +ur_result_t SanitizerInterceptor::partitionMemoryBuffer( + ur_mem_handle_t Buffer, ur_mem_flags_t Flags, + ur_buffer_create_type_t BufferCreateType, const ur_buffer_region_t *Region, + ur_mem_handle_t *OutBuffer) { + + ur_result_t result = + m_Dditable.Mem.pfnBufferPartition(Buffer, Flags, BufferCreateType, Region, OutBuffer); + + return result; +} + } // namespace ur_asan_layer diff --git a/source/loader/layers/sanitizer/asan/sanitizer_interceptor.hpp b/source/loader/layers/sanitizer/asan/sanitizer_interceptor.hpp index b7a54a0c9b..25bffbe3ce 100644 --- a/source/loader/layers/sanitizer/asan/sanitizer_interceptor.hpp +++ b/source/loader/layers/sanitizer/asan/sanitizer_interceptor.hpp @@ -31,10 +31,10 @@ struct USMAllocInfo { USMMemoryType Type; }; -enum class DeviceType { CPU, GPU_PVC, GPU_DG2 }; +enum class DeviceType { UNKNOWN, CPU, GPU_PVC, GPU_DG2 }; struct DeviceInfo { - ur_device_type_t Type; + DeviceType Type; size_t Alignment; uptr ShadowOffset; uptr ShadowOffsetEnd; @@ -97,11 +97,15 @@ class SanitizerInterceptor { void postLaunchKernel(ur_kernel_handle_t Kernel, ur_queue_handle_t Queue, ur_event_handle_t *Event, bool SetCallback = true); - void checkSanitizerError(ur_kernel_handle_t Kernel); ur_result_t createMemoryBuffer(ur_context_handle_t Context, ur_mem_flags_t Flags, size_t Size, const ur_buffer_properties_t *Properties, - ur_mem_handle_t *Buffer); + ur_mem_handle_t *OutBuffer); + ur_result_t partitionMemoryBuffer(ur_mem_handle_t Buffer, + ur_mem_flags_t Flags, + ur_buffer_create_type_t BufferCreateType, + const ur_buffer_region_t *Region, + ur_mem_handle_t *OutBuffer); ur_result_t addContext(ur_context_handle_t Context); ur_result_t addDevice(ur_context_handle_t Context, diff --git a/source/loader/layers/sanitizer/asan/ur_asnddi.cpp b/source/loader/layers/sanitizer/asan/ur_asnddi.cpp index 4018a4ea11..762c42761c 100644 --- a/source/loader/layers/sanitizer/asan/ur_asnddi.cpp +++ b/source/loader/layers/sanitizer/asan/ur_asnddi.cpp @@ -234,6 +234,28 @@ __urdlllocal ur_result_t UR_APICALL urMemBufferCreate( pProperties, phBuffer); } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urMemBufferPartition +__urdlllocal ur_result_t UR_APICALL urMemBufferPartition( + ur_mem_handle_t + hBuffer, ///< [in] handle of the buffer object to allocate from + ur_mem_flags_t flags, ///< [in] allocation and usage information flags + ur_buffer_create_type_t bufferCreateType, ///< [in] buffer creation type + const ur_buffer_region_t + *pRegion, ///< [in] pointer to buffer create region information + ur_mem_handle_t + *phMem ///< [out] pointer to the handle of sub buffer created +) { + auto pfnBufferPartition = context.urDdiTable.Mem.pfnBufferPartition; + + if (nullptr == pfnBufferPartition) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + + return context.interceptor->partitionMemoryBuffer( + hBuffer, flags, bufferCreateType, pRegion, phMem); +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Intercept function for urKernelSetArgLocal __urdlllocal ur_result_t UR_APICALL urKernelSetArgLocal( @@ -458,7 +480,7 @@ __urdlllocal ur_result_t UR_APICALL urGetMemProcAddrTable( // pDdiTable->pfnRelease = ur_asan_layer::urMemRelease; // dditable.pfnBufferPartition = pDdiTable->pfnBufferPartition; - // pDdiTable->pfnBufferPartition = ur_asan_layer::urMemBufferPartition; + pDdiTable->pfnBufferPartition = ur_asan_layer::urMemBufferPartition; // dditable.pfnGetNativeHandle = pDdiTable->pfnGetNativeHandle; // pDdiTable->pfnGetNativeHandle = ur_asan_layer::urMemGetNativeHandle; From 0a72ea9633189829703abc359d06c17b12ee708b Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Mon, 13 Nov 2023 22:24:50 -0800 Subject: [PATCH 20/72] UR: clean code --- .../sanitizer/asan/sanitizer_interceptor.cpp | 81 ----- .../sanitizer/asan/sanitizer_interceptor.hpp | 10 - .../layers/sanitizer/asan/ur_asnddi.cpp | 282 +----------------- 3 files changed, 13 insertions(+), 360 deletions(-) diff --git a/source/loader/layers/sanitizer/asan/sanitizer_interceptor.cpp b/source/loader/layers/sanitizer/asan/sanitizer_interceptor.cpp index 32103f292f..e2e2049523 100644 --- a/source/loader/layers/sanitizer/asan/sanitizer_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan/sanitizer_interceptor.cpp @@ -616,85 +616,4 @@ void SanitizerInterceptor::prepareLaunch(ur_queue_handle_t Queue, QueueInfo.LastEvent = LastEvent; } -ur_result_t SanitizerInterceptor::createMemoryBuffer( - ur_context_handle_t Context, ur_mem_flags_t Flags, size_t Size, - const ur_buffer_properties_t *Properties, ur_mem_handle_t *OutBuffer) { - auto &ContextInfo = getContextInfo(Context); - - constexpr size_t Alignment = ASAN_SHADOW_GRANULARITY; - - // Calcuate Size + RZSize - const uptr RZLog = ComputeRZLog(Size); - const uptr RZSize = RZLog2Size(RZLog); - const uptr RoundedSize = RoundUpTo(Size, Alignment); - const uptr NeededSize = RoundedSize + RZSize * 2; - - auto Result = m_Dditable.Mem.pfnBufferCreate(Context, Flags, NeededSize, - Properties, OutBuffer); - if (Result != UR_RESULT_SUCCESS) { - return Result; - } - - // Get Native Handle - ur_native_handle_t NativeHandle{}; - Result = m_Dditable.Mem.pfnGetNativeHandle(*OutBuffer, &NativeHandle); - if (Result != UR_RESULT_SUCCESS) { - return Result; - } - void *Allocated = (void *)NativeHandle; - - ur_device_handle_t Device; - Result = m_Dditable.USM.pfnGetMemAllocInfo( - Context, Allocated, UR_USM_ALLOC_INFO_DEVICE, - sizeof(ur_device_handle_t), &Device, nullptr); - - uptr AllocBegin = reinterpret_cast(Allocated); - uptr AllocEnd = AllocBegin + NeededSize; - uptr UserBegin = AllocBegin + RZSize; - if (!IsAligned(UserBegin, Alignment)) { - UserBegin = RoundUpTo(UserBegin, Alignment); - } - uptr UserEnd = UserBegin + Size; - assert(UserEnd <= AllocEnd); - - auto MemoryInfo = USMAllocInfo{AllocBegin, UserBegin, UserEnd, NeededSize, - USMMemoryType::MEM_BUFFER}; - - { - auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); - std::scoped_lock Guard(DeviceInfo.Mutex); - DeviceInfo.AllocInfos.emplace_back(MemoryInfo); - } - - { - std::scoped_lock Guard(ContextInfo.Mutex); - ContextInfo.AllocatedUSMMap[AllocBegin] = MemoryInfo; - } - - // Create a subbuffer which is surrounded by red zone - ur_buffer_region_t BufferRegion{UR_STRUCTURE_TYPE_BUFFER_REGION, nullptr, - UserBegin - AllocBegin, Size}; - ur_mem_handle_t SubBuffer; - Result = m_Dditable.Mem.pfnBufferPartition(*OutBuffer, Flags, - UR_BUFFER_CREATE_TYPE_REGION, - &BufferRegion, &SubBuffer); - if (Result != UR_RESULT_SUCCESS) { - return Result; - } - - *OutBuffer = SubBuffer; - return UR_RESULT_SUCCESS; -} - -ur_result_t SanitizerInterceptor::partitionMemoryBuffer( - ur_mem_handle_t Buffer, ur_mem_flags_t Flags, - ur_buffer_create_type_t BufferCreateType, const ur_buffer_region_t *Region, - ur_mem_handle_t *OutBuffer) { - - ur_result_t result = - m_Dditable.Mem.pfnBufferPartition(Buffer, Flags, BufferCreateType, Region, OutBuffer); - - return result; -} - } // namespace ur_asan_layer diff --git a/source/loader/layers/sanitizer/asan/sanitizer_interceptor.hpp b/source/loader/layers/sanitizer/asan/sanitizer_interceptor.hpp index 25bffbe3ce..3f20741e51 100644 --- a/source/loader/layers/sanitizer/asan/sanitizer_interceptor.hpp +++ b/source/loader/layers/sanitizer/asan/sanitizer_interceptor.hpp @@ -97,16 +97,6 @@ class SanitizerInterceptor { void postLaunchKernel(ur_kernel_handle_t Kernel, ur_queue_handle_t Queue, ur_event_handle_t *Event, bool SetCallback = true); - ur_result_t createMemoryBuffer(ur_context_handle_t Context, - ur_mem_flags_t Flags, size_t Size, - const ur_buffer_properties_t *Properties, - ur_mem_handle_t *OutBuffer); - ur_result_t partitionMemoryBuffer(ur_mem_handle_t Buffer, - ur_mem_flags_t Flags, - ur_buffer_create_type_t BufferCreateType, - const ur_buffer_region_t *Region, - ur_mem_handle_t *OutBuffer); - ur_result_t addContext(ur_context_handle_t Context); ur_result_t addDevice(ur_context_handle_t Context, ur_device_handle_t Device); diff --git a/source/loader/layers/sanitizer/asan/ur_asnddi.cpp b/source/loader/layers/sanitizer/asan/ur_asnddi.cpp index 762c42761c..ced6d4b328 100644 --- a/source/loader/layers/sanitizer/asan/ur_asnddi.cpp +++ b/source/loader/layers/sanitizer/asan/ur_asnddi.cpp @@ -219,96 +219,6 @@ __urdlllocal ur_result_t UR_APICALL urEnqueueKernelLaunch( return result; } -/////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urMemBufferCreate -__urdlllocal ur_result_t UR_APICALL urMemBufferCreate( - ur_context_handle_t hContext, ///< [in] handle of the context object - ur_mem_flags_t flags, ///< [in] allocation and usage information flags - size_t size, ///< [in] size in bytes of the memory object to be allocated - const ur_buffer_properties_t - *pProperties, ///< [in][optional] pointer to buffer creation properties - ur_mem_handle_t - *phBuffer ///< [out] pointer to handle of the memory buffer created -) { - return context.interceptor->createMemoryBuffer(hContext, flags, size, - pProperties, phBuffer); -} - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urMemBufferPartition -__urdlllocal ur_result_t UR_APICALL urMemBufferPartition( - ur_mem_handle_t - hBuffer, ///< [in] handle of the buffer object to allocate from - ur_mem_flags_t flags, ///< [in] allocation and usage information flags - ur_buffer_create_type_t bufferCreateType, ///< [in] buffer creation type - const ur_buffer_region_t - *pRegion, ///< [in] pointer to buffer create region information - ur_mem_handle_t - *phMem ///< [out] pointer to the handle of sub buffer created -) { - auto pfnBufferPartition = context.urDdiTable.Mem.pfnBufferPartition; - - if (nullptr == pfnBufferPartition) { - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - } - - return context.interceptor->partitionMemoryBuffer( - hBuffer, flags, bufferCreateType, pRegion, phMem); -} - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urKernelSetArgLocal -__urdlllocal ur_result_t UR_APICALL urKernelSetArgLocal( - ur_kernel_handle_t hKernel, ///< [in] handle of the kernel object - uint32_t argIndex, ///< [in] argument index in range [0, num args - 1] - size_t - argSize, ///< [in] size of the local buffer to be allocated by the runtime - const ur_kernel_arg_local_properties_t - *pProperties ///< [in][optional] pointer to local buffer properties. -) { - auto pfnSetArgLocal = context.urDdiTable.Kernel.pfnSetArgLocal; - - if (nullptr == pfnSetArgLocal) { - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - } - - context.logger.debug("=== urKernelSetArgLocal"); - - uptr rzLog = ComputeRZLog(argSize); - uptr rzSize = RZLog2Size(rzLog); - uptr roundedSize = RoundUpTo(argSize, ASAN_SHADOW_GRANULARITY); - uptr neededSize = roundedSize + rzSize; - - // TODO: How to update its shadow memory? - ur_result_t result = - pfnSetArgLocal(hKernel, argIndex, neededSize, pProperties); - - return result; -} - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urKernelSetArgMemObj -__urdlllocal ur_result_t UR_APICALL urKernelSetArgMemObj( - ur_kernel_handle_t hKernel, ///< [in] handle of the kernel object - uint32_t argIndex, ///< [in] argument index in range [0, num args - 1] - const ur_kernel_arg_mem_obj_properties_t - *pProperties, ///< [in][optional] pointer to Memory object properties. - ur_mem_handle_t hArgValue ///< [in][optional] handle of Memory object. -) { - auto pfnSetArgMemObj = context.urDdiTable.Kernel.pfnSetArgMemObj; - - if (nullptr == pfnSetArgMemObj) { - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - } - - context.logger.debug("=== urKernelSetArgMemObj"); - - ur_result_t result = - pfnSetArgMemObj(hKernel, argIndex, pProperties, hArgValue); - - return result; -} - /////////////////////////////////////////////////////////////////////////////// /// @brief Intercept function for urContextCreate __urdlllocal ur_result_t UR_APICALL urContextCreate( @@ -399,8 +309,6 @@ __urdlllocal ur_result_t UR_APICALL urGetContextProcAddrTable( ur_context_dditable_t *pDdiTable ///< [in,out] pointer to table of DDI function pointers ) { - auto &dditable = ur_asan_layer::context.urDdiTable.Context; - if (nullptr == pDdiTable) { return UR_RESULT_ERROR_INVALID_NULL_POINTER; } @@ -414,93 +322,11 @@ __urdlllocal ur_result_t UR_APICALL urGetContextProcAddrTable( ur_result_t result = UR_RESULT_SUCCESS; - dditable.pfnCreate = pDdiTable->pfnCreate; pDdiTable->pfnCreate = ur_asan_layer::urContextCreate; - // dditable.pfnRetain = pDdiTable->pfnRetain; - // pDdiTable->pfnRetain = ur_asan_layer::urContextRetain; - - // dditable.pfnRelease = pDdiTable->pfnRelease; - // pDdiTable->pfnRelease = ur_asan_layer::urContextRelease; - - // dditable.pfnGetInfo = pDdiTable->pfnGetInfo; - // pDdiTable->pfnGetInfo = ur_asan_layer::urContextGetInfo; - - // dditable.pfnGetNativeHandle = pDdiTable->pfnGetNativeHandle; - // pDdiTable->pfnGetNativeHandle = ur_asan_layer::urContextGetNativeHandle; - - dditable.pfnCreateWithNativeHandle = pDdiTable->pfnCreateWithNativeHandle; pDdiTable->pfnCreateWithNativeHandle = ur_asan_layer::urContextCreateWithNativeHandle; - // dditable.pfnSetExtendedDeleter = pDdiTable->pfnSetExtendedDeleter; - // pDdiTable->pfnSetExtendedDeleter = - // ur_asan_layer::urContextSetExtendedDeleter; - - return result; -} -/////////////////////////////////////////////////////////////////////////////// -/// @brief Exported function for filling application's Mem table -/// with current process' addresses -/// -/// @returns -/// - ::UR_RESULT_SUCCESS -/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// - ::UR_RESULT_ERROR_UNSUPPORTED_VERSION -__urdlllocal ur_result_t UR_APICALL urGetMemProcAddrTable( - ur_api_version_t version, ///< [in] API version requested - ur_mem_dditable_t - *pDdiTable ///< [in,out] pointer to table of DDI function pointers -) { - // auto &dditable = ur_asan_layer::context.urDdiTable.Mem; - - if (nullptr == pDdiTable) { - return UR_RESULT_ERROR_INVALID_NULL_POINTER; - } - - if (UR_MAJOR_VERSION(ur_asan_layer::context.version) != - UR_MAJOR_VERSION(version) || - UR_MINOR_VERSION(ur_asan_layer::context.version) > - UR_MINOR_VERSION(version)) { - return UR_RESULT_ERROR_UNSUPPORTED_VERSION; - } - - ur_result_t result = UR_RESULT_SUCCESS; - - // dditable.pfnImageCreate = pDdiTable->pfnImageCreate; - // pDdiTable->pfnImageCreate = ur_asan_layer::urMemImageCreate; - - // dditable.pfnBufferCreate = pDdiTable->pfnBufferCreate; - pDdiTable->pfnBufferCreate = ur_asan_layer::urMemBufferCreate; - - // dditable.pfnRetain = pDdiTable->pfnRetain; - // pDdiTable->pfnRetain = ur_asan_layer::urMemRetain; - - // dditable.pfnRelease = pDdiTable->pfnRelease; - // pDdiTable->pfnRelease = ur_asan_layer::urMemRelease; - - // dditable.pfnBufferPartition = pDdiTable->pfnBufferPartition; - pDdiTable->pfnBufferPartition = ur_asan_layer::urMemBufferPartition; - - // dditable.pfnGetNativeHandle = pDdiTable->pfnGetNativeHandle; - // pDdiTable->pfnGetNativeHandle = ur_asan_layer::urMemGetNativeHandle; - - // dditable.pfnBufferCreateWithNativeHandle = - // pDdiTable->pfnBufferCreateWithNativeHandle; - // pDdiTable->pfnBufferCreateWithNativeHandle = - // ur_asan_layer::urMemBufferCreateWithNativeHandle; - - // dditable.pfnImageCreateWithNativeHandle = - // pDdiTable->pfnImageCreateWithNativeHandle; - // pDdiTable->pfnImageCreateWithNativeHandle = - // ur_asan_layer::urMemImageCreateWithNativeHandle; - - // dditable.pfnGetInfo = pDdiTable->pfnGetInfo; - // pDdiTable->pfnGetInfo = ur_asan_layer::urMemGetInfo; - - // dditable.pfnImageGetInfo = pDdiTable->pfnImageGetInfo; - // pDdiTable->pfnImageGetInfo = ur_asan_layer::urMemImageGetInfo; - return result; } /////////////////////////////////////////////////////////////////////////////// @@ -516,8 +342,6 @@ __urdlllocal ur_result_t UR_APICALL urGetEnqueueProcAddrTable( ur_enqueue_dditable_t *pDdiTable ///< [in,out] pointer to table of DDI function pointers ) { - // auto &dditable = ur_asan_layer::context.urDdiTable.Enqueue; - if (nullptr == pDdiTable) { return UR_RESULT_ERROR_INVALID_NULL_POINTER; } @@ -531,7 +355,6 @@ __urdlllocal ur_result_t UR_APICALL urGetEnqueueProcAddrTable( ur_result_t result = UR_RESULT_SUCCESS; - // dditable.pfnKernelLaunch = pDdiTable->pfnKernelLaunch; pDdiTable->pfnKernelLaunch = ur_asan_layer::urEnqueueKernelLaunch; return result; @@ -549,8 +372,6 @@ __urdlllocal ur_result_t UR_APICALL urGetKernelProcAddrTable( ur_kernel_dditable_t *pDdiTable ///< [in,out] pointer to table of DDI function pointers ) { - // auto &dditable = ur_asan_layer::context.urDdiTable.Kernel; - if (nullptr == pDdiTable) { return UR_RESULT_ERROR_INVALID_NULL_POINTER; } @@ -564,10 +385,7 @@ __urdlllocal ur_result_t UR_APICALL urGetKernelProcAddrTable( ur_result_t result = UR_RESULT_SUCCESS; - // dditable.pfnCreate = pDdiTable->pfnCreate; pDdiTable->pfnCreate = ur_asan_layer::urKernelCreate; - // pDdiTable->pfnSetArgLocal = ur_asan_layer::urKernelSetArgLocal; - pDdiTable->pfnSetArgMemObj = ur_asan_layer::urKernelSetArgMemObj; return result; } @@ -584,8 +402,6 @@ __urdlllocal ur_result_t UR_APICALL urGetQueueProcAddrTable( ur_queue_dditable_t *pDdiTable ///< [in,out] pointer to table of DDI function pointers ) { - // auto &dditable = ur_asan_layer::context.urDdiTable.Queue; - if (nullptr == pDdiTable) { return UR_RESULT_ERROR_INVALID_NULL_POINTER; } @@ -599,7 +415,6 @@ __urdlllocal ur_result_t UR_APICALL urGetQueueProcAddrTable( ur_result_t result = UR_RESULT_SUCCESS; - // dditable.pfnCreate = pDdiTable->pfnCreate; pDdiTable->pfnCreate = ur_asan_layer::urQueueCreate; return result; @@ -617,8 +432,6 @@ __urdlllocal ur_result_t UR_APICALL urGetUSMProcAddrTable( ur_usm_dditable_t *pDdiTable ///< [in,out] pointer to table of DDI function pointers ) { - // auto &dditable = ur_asan_layer::context.urDdiTable.USM; - if (nullptr == pDdiTable) { return UR_RESULT_ERROR_INVALID_NULL_POINTER; } @@ -632,77 +445,8 @@ __urdlllocal ur_result_t UR_APICALL urGetUSMProcAddrTable( ur_result_t result = UR_RESULT_SUCCESS; - // dditable.pfnHostAlloc = pDdiTable->pfnHostAlloc; - // pDdiTable->pfnHostAlloc = ur_asan_layer::urUSMHostAlloc; - - // dditable.pfnDeviceAlloc = pDdiTable->pfnDeviceAlloc; pDdiTable->pfnDeviceAlloc = ur_asan_layer::urUSMDeviceAlloc; - // dditable.pfnSharedAlloc = pDdiTable->pfnSharedAlloc; - // pDdiTable->pfnSharedAlloc = ur_asan_layer::urUSMSharedAlloc; - - // dditable.pfnFree = pDdiTable->pfnFree; - // pDdiTable->pfnFree = ur_asan_layer::urUSMFree; - - return result; -} -/////////////////////////////////////////////////////////////////////////////// -/// @brief Exported function for filling application's Device table -/// with current process' addresses -/// -/// @returns -/// - ::UR_RESULT_SUCCESS -/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// - ::UR_RESULT_ERROR_UNSUPPORTED_VERSION -__urdlllocal ur_result_t UR_APICALL urGetDeviceProcAddrTable( - ur_api_version_t version, ///< [in] API version requested - ur_device_dditable_t - *pDdiTable ///< [in,out] pointer to table of DDI function pointers -) { - // auto &dditable = ur_asan_layer::context.urDdiTable.Device; - - if (nullptr == pDdiTable) { - return UR_RESULT_ERROR_INVALID_NULL_POINTER; - } - - if (UR_MAJOR_VERSION(ur_asan_layer::context.version) != - UR_MAJOR_VERSION(version) || - UR_MINOR_VERSION(ur_asan_layer::context.version) > - UR_MINOR_VERSION(version)) { - return UR_RESULT_ERROR_UNSUPPORTED_VERSION; - } - - ur_result_t result = UR_RESULT_SUCCESS; - - // dditable.pfnGet = pDdiTable->pfnGet; - // pDdiTable->pfnGet = ur_asan_layer::urDeviceGet; - - // dditable.pfnGetInfo = pDdiTable->pfnGetInfo; - // pDdiTable->pfnGetInfo = ur_asan_layer::urDeviceGetInfo; - - // dditable.pfnRetain = pDdiTable->pfnRetain; - // pDdiTable->pfnRetain = ur_asan_layer::urDeviceRetain; - - // dditable.pfnRelease = pDdiTable->pfnRelease; - // pDdiTable->pfnRelease = ur_asan_layer::urDeviceRelease; - - // dditable.pfnPartition = pDdiTable->pfnPartition; - // pDdiTable->pfnPartition = ur_asan_layer::urDevicePartition; - - // dditable.pfnSelectBinary = pDdiTable->pfnSelectBinary; - // pDdiTable->pfnSelectBinary = ur_asan_layer::urDeviceSelectBinary; - - // dditable.pfnGetNativeHandle = pDdiTable->pfnGetNativeHandle; - // pDdiTable->pfnGetNativeHandle = ur_asan_layer::urDeviceGetNativeHandle; - - // dditable.pfnCreateWithNativeHandle = pDdiTable->pfnCreateWithNativeHandle; - // pDdiTable->pfnCreateWithNativeHandle = - // ur_asan_layer::urDeviceCreateWithNativeHandle; - - // dditable.pfnGetGlobalTimestamps = pDdiTable->pfnGetGlobalTimestamps; - // pDdiTable->pfnGetGlobalTimestamps = - // ur_asan_layer::urDeviceGetGlobalTimestamps; - return result; } @@ -710,36 +454,36 @@ ur_result_t context_t::init(ur_dditable_t *dditable, const std::set &enabledLayerNames) { ur_result_t result = UR_RESULT_SUCCESS; - context.logger.info("ur_asan_layer init"); + if (!enabledLayerNames.count(name)) { + return result; + } - // if (!enabledLayerNames.count(name)) { - // return result; - // } + // FIXME: Just copy needed APIs? + urDdiTable = *dditable; if (UR_RESULT_SUCCESS == result) { - // FIXME: Just copy needed APIs? - urDdiTable = *dditable; - result = ur_asan_layer::urGetContextProcAddrTable( UR_API_VERSION_CURRENT, &dditable->Context); + } - // result = ur_asan_layer::urGetDeviceProcAddrTable( - // UR_API_VERSION_CURRENT, &dditable->Device); - + if (UR_RESULT_SUCCESS == result) { result = ur_asan_layer::urGetEnqueueProcAddrTable( UR_API_VERSION_CURRENT, &dditable->Enqueue); + } + if (UR_RESULT_SUCCESS == result) { result = ur_asan_layer::urGetKernelProcAddrTable(UR_API_VERSION_CURRENT, &dditable->Kernel); + } + if (UR_RESULT_SUCCESS == result) { result = ur_asan_layer::urGetQueueProcAddrTable(UR_API_VERSION_CURRENT, &dditable->Queue); + } + if (UR_RESULT_SUCCESS == result) { result = ur_asan_layer::urGetUSMProcAddrTable(UR_API_VERSION_CURRENT, &dditable->USM); - - result = ur_asan_layer::urGetMemProcAddrTable(UR_API_VERSION_CURRENT, - &dditable->Mem); } return result; From 8fa86dd4d1f159ddef2a08c2328935164f12bf60 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Mon, 13 Nov 2023 22:26:25 -0800 Subject: [PATCH 21/72] UR: clean code --- source/loader/layers/sanitizer/asan/ur_asan_layer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/loader/layers/sanitizer/asan/ur_asan_layer.cpp b/source/loader/layers/sanitizer/asan/ur_asan_layer.cpp index b352ddbf6a..05d2c23193 100644 --- a/source/loader/layers/sanitizer/asan/ur_asan_layer.cpp +++ b/source/loader/layers/sanitizer/asan/ur_asan_layer.cpp @@ -22,7 +22,7 @@ context_t context; /////////////////////////////////////////////////////////////////////////////// context_t::context_t() : interceptor(new SanitizerInterceptor(urDdiTable)), - logger(logger::create_logger("sanitizer")) {} + logger(logger::create_logger("asan")) {} bool context_t::isAvailable() const { return true; } From d9a51a79a0a80650b2b5dc070fe47ad8bb6e076c Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Mon, 13 Nov 2023 22:29:59 -0800 Subject: [PATCH 22/72] Revert "an -> a" This reverts commit 84a3afa631728a72470d42fef795aabecb6a7dd4. --- source/adapters/level_zero/virtual_mem.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/adapters/level_zero/virtual_mem.cpp b/source/adapters/level_zero/virtual_mem.cpp index 545f9fde54..e90aec45de 100644 --- a/source/adapters/level_zero/virtual_mem.cpp +++ b/source/adapters/level_zero/virtual_mem.cpp @@ -22,7 +22,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemGranularityGetInfo( switch (propName) { case UR_VIRTUAL_MEM_GRANULARITY_INFO_MINIMUM: case UR_VIRTUAL_MEM_GRANULARITY_INFO_RECOMMENDED: { - // For L0 the minimum and recommended granularity is the same. We use a + // For L0 the minimum and recommended granularity is the same. We use an // memory size of 1 byte to get the actual granularity instead of the // aligned size. size_t PageSize; From 730bf453b611011af84334f5cca208b8f468ca10 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Mon, 13 Nov 2023 22:30:11 -0800 Subject: [PATCH 23/72] Revert "Fix wrongful use of UR_ASSERT in cuda implementation" This reverts commit 336ce895dcf3f0f2f9a96e81ac372f86f92cbc86. --- source/adapters/cuda/physical_mem.cpp | 3 ++- source/adapters/cuda/physical_mem.hpp | 8 +++++--- source/adapters/cuda/virtual_mem.cpp | 13 ++++++++----- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/source/adapters/cuda/physical_mem.cpp b/source/adapters/cuda/physical_mem.cpp index 444d492aa3..177bc2234d 100644 --- a/source/adapters/cuda/physical_mem.cpp +++ b/source/adapters/cuda/physical_mem.cpp @@ -23,7 +23,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urPhysicalMemCreate( CUmemAllocationProp AllocProps = {}; AllocProps.location.type = CU_MEM_LOCATION_TYPE_DEVICE; AllocProps.type = CU_MEM_ALLOCATION_TYPE_PINNED; - UR_CHECK_ERROR(GetDeviceOrdinal(hDevice, AllocProps.location.id)); + UR_ASSERT(GetDeviceOrdinal(hDevice, AllocProps.location.id), + UR_RESULT_ERROR_INVALID_DEVICE); CUmemGenericAllocationHandle ResHandle; UR_CHECK_ERROR(cuMemCreate(&ResHandle, size, &AllocProps, 0)); diff --git a/source/adapters/cuda/physical_mem.hpp b/source/adapters/cuda/physical_mem.hpp index 0ce332e112..2b0dc029d5 100644 --- a/source/adapters/cuda/physical_mem.hpp +++ b/source/adapters/cuda/physical_mem.hpp @@ -50,12 +50,14 @@ inline ur_result_t GetDeviceOrdinal(ur_device_handle_t Device, int &Ordinal) { ur_adapter_handle_t AdapterHandle = &adapter; // Get list of platforms uint32_t NumPlatforms; - UR_CHECK_ERROR(urPlatformGet(&AdapterHandle, 1, 0, nullptr, &NumPlatforms)); + UR_ASSERT(urPlatformGet(&AdapterHandle, 1, 0, nullptr, &NumPlatforms), + UR_RESULT_ERROR_INVALID_ARGUMENT); UR_ASSERT(NumPlatforms, UR_RESULT_ERROR_UNKNOWN); std::vector Platforms{NumPlatforms}; - UR_CHECK_ERROR(urPlatformGet(&AdapterHandle, 1, NumPlatforms, - Platforms.data(), nullptr)); + UR_ASSERT( + urPlatformGet(&AdapterHandle, 1, NumPlatforms, Platforms.data(), nullptr), + UR_RESULT_ERROR_INVALID_ARGUMENT); // Ordinal corresponds to the platform ID as each device has its own platform. CUdevice NativeDevice = Device->get(); diff --git a/source/adapters/cuda/virtual_mem.cpp b/source/adapters/cuda/virtual_mem.cpp index 9c37dda4fb..f49da3132e 100644 --- a/source/adapters/cuda/virtual_mem.cpp +++ b/source/adapters/cuda/virtual_mem.cpp @@ -33,7 +33,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemGranularityGetInfo( CUmemAllocationProp AllocProps = {}; AllocProps.location.type = CU_MEM_LOCATION_TYPE_DEVICE; AllocProps.type = CU_MEM_ALLOCATION_TYPE_PINNED; - UR_CHECK_ERROR(GetDeviceOrdinal(hDevice, AllocProps.location.id)); + UR_ASSERT(GetDeviceOrdinal(hDevice, AllocProps.location.id), + UR_RESULT_ERROR_INVALID_DEVICE); size_t Granularity; UR_CHECK_ERROR( @@ -77,8 +78,8 @@ urVirtualMemSetAccess(ur_context_handle_t hContext, const void *pStart, // TODO: When contexts support multiple devices, we should create a descriptor // for each. We may also introduce a variant of this function with a // specific device. - UR_CHECK_ERROR( - GetDeviceOrdinal(hContext->getDevice(), AccessDesc.location.id)); + UR_ASSERT(GetDeviceOrdinal(hContext->getDevice(), AccessDesc.location.id), + UR_RESULT_ERROR_INVALID_DEVICE); ScopedContext Active(hContext); UR_CHECK_ERROR(cuMemSetAccess((CUdeviceptr)pStart, size, &AccessDesc, 1)); @@ -93,7 +94,8 @@ urVirtualMemMap(ur_context_handle_t hContext, const void *pStart, size_t size, UR_CHECK_ERROR( cuMemMap((CUdeviceptr)pStart, size, offset, hPhysicalMem->get(), 0)); if (flags) - UR_CHECK_ERROR(urVirtualMemSetAccess(hContext, pStart, size, flags)); + UR_ASSERT(urVirtualMemSetAccess(hContext, pStart, size, flags), + UR_RESULT_ERROR_INVALID_ARGUMENT); return UR_RESULT_SUCCESS; } @@ -115,7 +117,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemGetInfo( case UR_VIRTUAL_MEM_INFO_ACCESS_MODE: { CUmemLocation MemLocation = {}; MemLocation.type = CU_MEM_LOCATION_TYPE_DEVICE; - UR_CHECK_ERROR(GetDeviceOrdinal(hContext->getDevice(), MemLocation.id)); + UR_ASSERT(GetDeviceOrdinal(hContext->getDevice(), MemLocation.id), + UR_RESULT_ERROR_INVALID_DEVICE); unsigned long long CuAccessFlags; UR_CHECK_ERROR( From fed541a00c0c4bdf4a1979229a9dd27091b23d15 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Mon, 13 Nov 2023 23:34:10 -0800 Subject: [PATCH 24/72] UR: refactor --- source/loader/CMakeLists.txt | 13 +- ...r_interceptor.cpp => asan_interceptor.cpp} | 21 ++- ...r_interceptor.hpp => asan_interceptor.hpp} | 28 ++-- .../sanitizer/{asan/common.h => common.hpp} | 57 ++------ .../{asan => }/device_sanitizer_report.hpp | 7 +- .../ur_asan_layer.cpp => ur_san_layer.cpp} | 14 +- .../ur_asan_layer.hpp => ur_san_layer.hpp} | 23 +-- .../{asan/ur_asnddi.cpp => ur_sanddi.cpp} | 136 +++++------------- source/loader/ur_lib.hpp | 4 +- 9 files changed, 113 insertions(+), 190 deletions(-) rename source/loader/layers/sanitizer/{asan/sanitizer_interceptor.cpp => asan_interceptor.cpp} (98%) rename source/loader/layers/sanitizer/{asan/sanitizer_interceptor.hpp => asan_interceptor.hpp} (90%) rename source/loader/layers/sanitizer/{asan/common.h => common.hpp} (89%) rename source/loader/layers/sanitizer/{asan => }/device_sanitizer_report.hpp (94%) rename source/loader/layers/sanitizer/{asan/ur_asan_layer.cpp => ur_san_layer.cpp} (74%) rename source/loader/layers/sanitizer/{asan/ur_asan_layer.hpp => ur_san_layer.hpp} (69%) rename source/loader/layers/sanitizer/{asan/ur_asnddi.cpp => ur_sanddi.cpp} (78%) diff --git a/source/loader/CMakeLists.txt b/source/loader/CMakeLists.txt index 5439fff6ac..43978c5aa6 100644 --- a/source/loader/CMakeLists.txt +++ b/source/loader/CMakeLists.txt @@ -103,13 +103,14 @@ endif() if(UR_ENABLE_SANITIZER) target_sources(ur_loader PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/asan/device_sanitizer_report.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/asan/sanitizer_interceptor.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/asan/sanitizer_interceptor.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/asan/ur_asan_layer.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/asan/ur_asan_layer.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/asan/ur_asnddi.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../ur/ur.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/asan_interceptor.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/asan_interceptor.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/common.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/device_sanitizer_report.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/ur_san_layer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/ur_san_layer.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/ur_sanddi.cpp ) target_include_directories(ur_loader PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../" diff --git a/source/loader/layers/sanitizer/asan/sanitizer_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp similarity index 98% rename from source/loader/layers/sanitizer/asan/sanitizer_interceptor.cpp rename to source/loader/layers/sanitizer/asan_interceptor.cpp index e2e2049523..38e10cc257 100644 --- a/source/loader/layers/sanitizer/asan/sanitizer_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -5,16 +5,27 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// - -#include "sanitizer_interceptor.hpp" +/* + * + * Copyright (C) 2023 Intel Corporation + * + * Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. + * See LICENSE.TXT + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + * @file ur_san_layer.cpp + * + */ + +#include "asan_interceptor.hpp" #include "device_sanitizer_report.hpp" -#include "ur_asan_layer.hpp" +#include "ur_san_layer.hpp" #include #include #include -namespace ur_asan_layer { +namespace ur_san_layer { namespace { @@ -616,4 +627,4 @@ void SanitizerInterceptor::prepareLaunch(ur_queue_handle_t Queue, QueueInfo.LastEvent = LastEvent; } -} // namespace ur_asan_layer +} // namespace ur_san_layer diff --git a/source/loader/layers/sanitizer/asan/sanitizer_interceptor.hpp b/source/loader/layers/sanitizer/asan_interceptor.hpp similarity index 90% rename from source/loader/layers/sanitizer/asan/sanitizer_interceptor.hpp rename to source/loader/layers/sanitizer/asan_interceptor.hpp index 3f20741e51..de2ddd62ba 100644 --- a/source/loader/layers/sanitizer/asan/sanitizer_interceptor.hpp +++ b/source/loader/layers/sanitizer/asan_interceptor.hpp @@ -1,25 +1,25 @@ -//==---------- sanitizer_interceptor.hpp - Sanitizer interceptor -----------==// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - +/* + * + * Copyright (C) 2023 Intel Corporation + * + * Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. + * See LICENSE.TXT + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + * @file ur_san_layer.cpp + * + */ #pragma once -#include "common.h" +#include "common.hpp" #include -#include #include #include #include -#include -#include #include -namespace ur_asan_layer { +namespace ur_san_layer { enum USMMemoryType { DEVICE, SHARE, HOST, MEM_BUFFER }; @@ -144,4 +144,4 @@ class SanitizerInterceptor { ur_dditable_t &m_Dditable; }; -} // namespace ur_asan_layer +} // namespace ur_san_layer diff --git a/source/loader/layers/sanitizer/asan/common.h b/source/loader/layers/sanitizer/common.hpp similarity index 89% rename from source/loader/layers/sanitizer/asan/common.h rename to source/loader/layers/sanitizer/common.hpp index 968ec52c51..cedc29ccd1 100644 --- a/source/loader/layers/sanitizer/asan/common.h +++ b/source/loader/layers/sanitizer/common.hpp @@ -1,11 +1,21 @@ -//==---------- sanitizer_interceptor.hpp - Sanitizer interceptor -----------==// +//==---------- common.hpp - Device Sanitizer -----------==// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// - +/* + * + * Copyright (C) 2023 Intel Corporation + * + * Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. + * See LICENSE.TXT + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + * @file common.hpp + * + */ #pragma once #include @@ -14,7 +24,7 @@ #include "ur_ddi.h" #include -namespace ur_asan_layer { +namespace ur_san_layer { typedef uintptr_t uptr; typedef unsigned char u8; @@ -41,30 +51,6 @@ inline constexpr bool IsAligned(uptr a, uptr alignment) { return (a & (alignment - 1)) == 0; } -/* -inline uptr LeastSignificantSetBitIndex(uptr x) { - // CHECK_NE(x, 0U); - unsigned long up; -#if !SANITIZER_WINDOWS || defined(__clang__) || defined(__GNUC__) -#ifdef _WIN64 - up = __builtin_ctzll(x); -#else - up = __builtin_ctzl(x); -#endif -#elif defined(_WIN64) - _BitScanForward64(&up, x); -#else - _BitScanForward(&up, x); -#endif - return up; -} - -inline uptr Log2(uptr x) { - // CHECK(IsPowerOfTwo(x)); - return LeastSignificantSetBitIndex(x); -} -*/ - // Valid redzone sizes are 16, 32, 64, ... 2048, so we encode them in 3 bits. // We use adaptive redzones: for larger allocation larger redzones are used. inline constexpr u32 RZLog2Size(u32 rz_log) { @@ -72,17 +58,6 @@ inline constexpr u32 RZLog2Size(u32 rz_log) { return 16 << rz_log; } -/* -static u32 RZSize2Log(u32 rz_size) { - // CHECK_GE(rz_size, 16); - // CHECK_LE(rz_size, 2048); - // CHECK(IsPowerOfTwo(rz_size)); - u32 res = Log2(rz_size) - 4; - // CHECK_EQ(rz_size, RZLog2Size(res)); - return res; -} -*/ - inline constexpr uptr ComputeRZLog(uptr user_requested_size) { u32 rz_log = user_requested_size <= 64 - 16 ? 0 : user_requested_size <= 128 - 32 ? 1 @@ -92,10 +67,6 @@ inline constexpr uptr ComputeRZLog(uptr user_requested_size) { : user_requested_size <= (1 << 15) - 512 ? 5 : user_requested_size <= (1 << 16) - 1024 ? 6 : 7; - // u32 hdr_log = RZSize2Log(RoundUpToPowerOfTwo(sizeof(ChunkHeader))); - // u32 min_log = RZSize2Log(atomic_load(&min_redzone, memory_order_acquire)); - // u32 max_log = RZSize2Log(atomic_load(&max_redzone, memory_order_acquire)); - // return Min(Max(rz_log, Max(min_log, hdr_log)), Max(max_log, hdr_log)); return rz_log; } @@ -259,4 +230,4 @@ static auto getUrResultString = [](ur_result_t Result) { return Result; \ } -} // namespace ur_asan_layer +} // namespace ur_san_layer diff --git a/source/loader/layers/sanitizer/asan/device_sanitizer_report.hpp b/source/loader/layers/sanitizer/device_sanitizer_report.hpp similarity index 94% rename from source/loader/layers/sanitizer/asan/device_sanitizer_report.hpp rename to source/loader/layers/sanitizer/device_sanitizer_report.hpp index 9ec6a69842..04318d280f 100644 --- a/source/loader/layers/sanitizer/asan/device_sanitizer_report.hpp +++ b/source/loader/layers/sanitizer/device_sanitizer_report.hpp @@ -1,16 +1,15 @@ -//==---------- sanitizer_interceptor.hpp - Sanitizer interceptor -----------==// +//==---------- device_sanitizer_report.hpp - Device Sanitizer -----------==// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// - #pragma once #include -namespace ur_asan_layer { +namespace ur_san_layer { enum class DeviceSanitizerErrorType : int32_t { UNKNOWN, @@ -90,4 +89,4 @@ const char *DeviceSanitizerFormat(DeviceSanitizerErrorType ErrorType) { } } -} // namespace ur_asan_layer +} // namespace ur_san_layer diff --git a/source/loader/layers/sanitizer/asan/ur_asan_layer.cpp b/source/loader/layers/sanitizer/ur_san_layer.cpp similarity index 74% rename from source/loader/layers/sanitizer/asan/ur_asan_layer.cpp rename to source/loader/layers/sanitizer/ur_san_layer.cpp index 05d2c23193..bfec1e3541 100644 --- a/source/loader/layers/sanitizer/asan/ur_asan_layer.cpp +++ b/source/loader/layers/sanitizer/ur_san_layer.cpp @@ -6,26 +6,24 @@ * See LICENSE.TXT * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception * - * @file ur_asan_layer.cpp + * @file ur_san_layer.cpp * */ -#include "ur_asan_layer.hpp" -#include "sanitizer_interceptor.hpp" +#include "ur_san_layer.hpp" +#include "asan_interceptor.hpp" #include "ur_api.h" #include "ur_util.hpp" -#include - -namespace ur_asan_layer { +namespace ur_san_layer { context_t context; /////////////////////////////////////////////////////////////////////////////// context_t::context_t() : interceptor(new SanitizerInterceptor(urDdiTable)), - logger(logger::create_logger("asan")) {} + logger(logger::create_logger("sanitizer")) {} bool context_t::isAvailable() const { return true; } /////////////////////////////////////////////////////////////////////////////// context_t::~context_t() {} -} // namespace ur_asan_layer +} // namespace ur_san_layer diff --git a/source/loader/layers/sanitizer/asan/ur_asan_layer.hpp b/source/loader/layers/sanitizer/ur_san_layer.hpp similarity index 69% rename from source/loader/layers/sanitizer/asan/ur_asan_layer.hpp rename to source/loader/layers/sanitizer/ur_san_layer.hpp index d92eb5839a..e9ec44e442 100644 --- a/source/loader/layers/sanitizer/asan/ur_asan_layer.hpp +++ b/source/loader/layers/sanitizer/ur_san_layer.hpp @@ -6,7 +6,7 @@ * See LICENSE.TXT * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception * - * @file ur_asan_layer.h + * @file ur_san_layer.hpp * */ @@ -18,33 +18,40 @@ #include "ur_proxy_layer.hpp" #include "ur_util.hpp" -#define SANITIZER_COMP_NAME "address sanitizer layer" +#define SANITIZER_COMP_NAME "sanitizer layer" -namespace ur_asan_layer { +namespace ur_san_layer { class SanitizerInterceptor; +enum class SanitizerType { + None, + AddressSanitizer, + MemorySanitizer, + ThreadSanitizer, +}; + /////////////////////////////////////////////////////////////////////////////// class __urdlllocal context_t : public proxy_layer_context_t { public: ur_dditable_t urDdiTable = {}; SanitizerInterceptor *interceptor = {}; logger::Logger logger; + SanitizerType enabledType = SanitizerType::None; context_t(); ~context_t(); bool isAvailable() const override; - std::vector getNames() const override { return {name}; } + std::vector getNames() const override { + return {"UR_LAYER_ASAN", "UR_LAYER_MSAN", "UR_LAYER_TSAN"}; + } ur_result_t init(ur_dditable_t *dditable, const std::set &enabledLayerNames) override; - - private: - const std::string name = "UR_LAYER_ASAN"; }; extern context_t context; -} // namespace ur_asan_layer +} // namespace ur_san_layer #endif /* UR_sanitizer_LAYER_H */ diff --git a/source/loader/layers/sanitizer/asan/ur_asnddi.cpp b/source/loader/layers/sanitizer/ur_sanddi.cpp similarity index 78% rename from source/loader/layers/sanitizer/asan/ur_asnddi.cpp rename to source/loader/layers/sanitizer/ur_sanddi.cpp index ced6d4b328..d75d0d891d 100644 --- a/source/loader/layers/sanitizer/asan/ur_asnddi.cpp +++ b/source/loader/layers/sanitizer/ur_sanddi.cpp @@ -6,18 +6,14 @@ * See LICENSE.TXT * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception * - * @file ur_trcddi.cpp + * @file ur_sanddi.cpp * */ -#include "common.h" -#include "sanitizer_interceptor.hpp" -#include "ur_asan_layer.hpp" +#include "asan_interceptor.hpp" +#include "ur_san_layer.hpp" -#include -#include - -namespace ur_asan_layer { +namespace ur_san_layer { /////////////////////////////////////////////////////////////////////////////// /// @brief Intercept function for urUSMHostAlloc @@ -37,8 +33,6 @@ __urdlllocal ur_result_t UR_APICALL urUSMHostAlloc( return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } - context.logger.debug("=== urUSMHostAlloc"); - return context.interceptor->allocateMemory( hContext, nullptr, pUSMDesc, pool, size, ppMem, USMMemoryType::HOST); } @@ -62,8 +56,6 @@ __urdlllocal ur_result_t UR_APICALL urUSMDeviceAlloc( return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } - context.logger.debug("=== urUSMDeviceAlloc"); - return context.interceptor->allocateMemory( hContext, hDevice, pUSMDesc, pool, size, ppMem, USMMemoryType::DEVICE); } @@ -87,8 +79,6 @@ __urdlllocal ur_result_t UR_APICALL urUSMSharedAlloc( return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } - context.logger.debug("=== urUSMSharedAlloc"); - return context.interceptor->allocateMemory( hContext, hDevice, pUSMDesc, pool, size, ppMem, USMMemoryType::SHARE); } @@ -105,34 +95,9 @@ __urdlllocal ur_result_t UR_APICALL urUSMFree( return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } - context.logger.debug("=== urUSMFree"); - return context.interceptor->releaseMemory(hContext, pMem); } -/////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urKernelCreate -__urdlllocal ur_result_t UR_APICALL urKernelCreate( - ur_program_handle_t hProgram, ///< [in] handle of the program instance - const char *pKernelName, ///< [in] pointer to null-terminated string. - ur_kernel_handle_t - *phKernel ///< [out] pointer to handle of kernel object created. -) { - auto pfnCreate = context.urDdiTable.Kernel.pfnCreate; - - if (nullptr == pfnCreate) { - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - } - // context.logger.debug("=== urKernelCreate"); - - ur_result_t result = pfnCreate(hProgram, pKernelName, phKernel); - // if (result == UR_RESULT_SUCCESS) { - // context.interceptor->addKernel(hProgram, *phKernel); - // } - - return result; -} - /////////////////////////////////////////////////////////////////////////////// /// @brief Intercept function for urQueueCreate __urdlllocal ur_result_t UR_APICALL urQueueCreate( @@ -148,7 +113,6 @@ __urdlllocal ur_result_t UR_APICALL urQueueCreate( if (nullptr == pfnCreate) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } - // context.logger.debug("=== urQueueCreate"); ur_result_t result = pfnCreate(hContext, hDevice, pProperties, phQueue); if (result == UR_RESULT_SUCCESS) { @@ -195,7 +159,6 @@ __urdlllocal ur_result_t UR_APICALL urEnqueueKernelLaunch( return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } - context.logger.debug("=== urEnqueueKernelLaunch"); ur_event_handle_t lk_event{}; std::vector events(numEventsInWaitList + 1); for (unsigned i = 0; i < numEventsInWaitList; ++i) { @@ -313,19 +276,19 @@ __urdlllocal ur_result_t UR_APICALL urGetContextProcAddrTable( return UR_RESULT_ERROR_INVALID_NULL_POINTER; } - if (UR_MAJOR_VERSION(ur_asan_layer::context.version) != + if (UR_MAJOR_VERSION(ur_san_layer::context.version) != UR_MAJOR_VERSION(version) || - UR_MINOR_VERSION(ur_asan_layer::context.version) > + UR_MINOR_VERSION(ur_san_layer::context.version) > UR_MINOR_VERSION(version)) { return UR_RESULT_ERROR_UNSUPPORTED_VERSION; } ur_result_t result = UR_RESULT_SUCCESS; - pDdiTable->pfnCreate = ur_asan_layer::urContextCreate; + pDdiTable->pfnCreate = ur_san_layer::urContextCreate; pDdiTable->pfnCreateWithNativeHandle = - ur_asan_layer::urContextCreateWithNativeHandle; + ur_san_layer::urContextCreateWithNativeHandle; return result; } @@ -346,46 +309,16 @@ __urdlllocal ur_result_t UR_APICALL urGetEnqueueProcAddrTable( return UR_RESULT_ERROR_INVALID_NULL_POINTER; } - if (UR_MAJOR_VERSION(ur_asan_layer::context.version) != - UR_MAJOR_VERSION(version) || - UR_MINOR_VERSION(ur_asan_layer::context.version) > - UR_MINOR_VERSION(version)) { - return UR_RESULT_ERROR_UNSUPPORTED_VERSION; - } - - ur_result_t result = UR_RESULT_SUCCESS; - - pDdiTable->pfnKernelLaunch = ur_asan_layer::urEnqueueKernelLaunch; - - return result; -} -/////////////////////////////////////////////////////////////////////////////// -/// @brief Exported function for filling application's Kernel table -/// with current process' addresses -/// -/// @returns -/// - ::UR_RESULT_SUCCESS -/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// - ::UR_RESULT_ERROR_UNSUPPORTED_VERSION -__urdlllocal ur_result_t UR_APICALL urGetKernelProcAddrTable( - ur_api_version_t version, ///< [in] API version requested - ur_kernel_dditable_t - *pDdiTable ///< [in,out] pointer to table of DDI function pointers -) { - if (nullptr == pDdiTable) { - return UR_RESULT_ERROR_INVALID_NULL_POINTER; - } - - if (UR_MAJOR_VERSION(ur_asan_layer::context.version) != + if (UR_MAJOR_VERSION(ur_san_layer::context.version) != UR_MAJOR_VERSION(version) || - UR_MINOR_VERSION(ur_asan_layer::context.version) > + UR_MINOR_VERSION(ur_san_layer::context.version) > UR_MINOR_VERSION(version)) { return UR_RESULT_ERROR_UNSUPPORTED_VERSION; } ur_result_t result = UR_RESULT_SUCCESS; - pDdiTable->pfnCreate = ur_asan_layer::urKernelCreate; + pDdiTable->pfnKernelLaunch = ur_san_layer::urEnqueueKernelLaunch; return result; } @@ -406,16 +339,16 @@ __urdlllocal ur_result_t UR_APICALL urGetQueueProcAddrTable( return UR_RESULT_ERROR_INVALID_NULL_POINTER; } - if (UR_MAJOR_VERSION(ur_asan_layer::context.version) != + if (UR_MAJOR_VERSION(ur_san_layer::context.version) != UR_MAJOR_VERSION(version) || - UR_MINOR_VERSION(ur_asan_layer::context.version) > + UR_MINOR_VERSION(ur_san_layer::context.version) > UR_MINOR_VERSION(version)) { return UR_RESULT_ERROR_UNSUPPORTED_VERSION; } ur_result_t result = UR_RESULT_SUCCESS; - pDdiTable->pfnCreate = ur_asan_layer::urQueueCreate; + pDdiTable->pfnCreate = ur_san_layer::urQueueCreate; return result; } @@ -436,16 +369,16 @@ __urdlllocal ur_result_t UR_APICALL urGetUSMProcAddrTable( return UR_RESULT_ERROR_INVALID_NULL_POINTER; } - if (UR_MAJOR_VERSION(ur_asan_layer::context.version) != + if (UR_MAJOR_VERSION(ur_san_layer::context.version) != UR_MAJOR_VERSION(version) || - UR_MINOR_VERSION(ur_asan_layer::context.version) > + UR_MINOR_VERSION(ur_san_layer::context.version) > UR_MINOR_VERSION(version)) { return UR_RESULT_ERROR_UNSUPPORTED_VERSION; } ur_result_t result = UR_RESULT_SUCCESS; - pDdiTable->pfnDeviceAlloc = ur_asan_layer::urUSMDeviceAlloc; + pDdiTable->pfnDeviceAlloc = ur_san_layer::urUSMDeviceAlloc; return result; } @@ -454,38 +387,41 @@ ur_result_t context_t::init(ur_dditable_t *dditable, const std::set &enabledLayerNames) { ur_result_t result = UR_RESULT_SUCCESS; - if (!enabledLayerNames.count(name)) { + if (enabledLayerNames.count("UR_LAYER_ASAN")) { + context.enabledType = SanitizerType::AddressSanitizer; + } else if (enabledLayerNames.count("UR_LAYER_MSAN")) { + context.enabledType = SanitizerType::MemorySanitizer; + } else if (enabledLayerNames.count("UR_LAYER_TSAN")) { + context.enabledType = SanitizerType::ThreadSanitizer; + } + + // Only support AddressSanitizer now + if (context.enabledType != SanitizerType::AddressSanitizer) { return result; } // FIXME: Just copy needed APIs? urDdiTable = *dditable; - - if (UR_RESULT_SUCCESS == result) { - result = ur_asan_layer::urGetContextProcAddrTable( - UR_API_VERSION_CURRENT, &dditable->Context); - } - if (UR_RESULT_SUCCESS == result) { - result = ur_asan_layer::urGetEnqueueProcAddrTable( - UR_API_VERSION_CURRENT, &dditable->Enqueue); + result = ur_san_layer::urGetContextProcAddrTable(UR_API_VERSION_CURRENT, + &dditable->Context); } if (UR_RESULT_SUCCESS == result) { - result = ur_asan_layer::urGetKernelProcAddrTable(UR_API_VERSION_CURRENT, - &dditable->Kernel); + result = ur_san_layer::urGetEnqueueProcAddrTable(UR_API_VERSION_CURRENT, + &dditable->Enqueue); } if (UR_RESULT_SUCCESS == result) { - result = ur_asan_layer::urGetQueueProcAddrTable(UR_API_VERSION_CURRENT, - &dditable->Queue); + result = ur_san_layer::urGetQueueProcAddrTable(UR_API_VERSION_CURRENT, + &dditable->Queue); } if (UR_RESULT_SUCCESS == result) { - result = ur_asan_layer::urGetUSMProcAddrTable(UR_API_VERSION_CURRENT, - &dditable->USM); + result = ur_san_layer::urGetUSMProcAddrTable(UR_API_VERSION_CURRENT, + &dditable->USM); } return result; } -} // namespace ur_asan_layer +} // namespace ur_san_layer diff --git a/source/loader/ur_lib.hpp b/source/loader/ur_lib.hpp index f754d214f2..d95db8771d 100644 --- a/source/loader/ur_lib.hpp +++ b/source/loader/ur_lib.hpp @@ -23,7 +23,7 @@ #include "tracing/ur_tracing_layer.hpp" #endif #if UR_ENABLE_SANITIZER -#include "sanitizer/asan/ur_asan_layer.hpp" +#include "sanitizer/ur_san_layer.hpp" #endif #include @@ -72,7 +72,7 @@ class __urdlllocal context_t { &ur_tracing_layer::context, #endif #if UR_ENABLE_SANITIZER - &ur_asan_layer::context + &ur_san_layer::context #endif }; std::string availableLayers; From ad2b1a769931d80a73eb1811d0138cf204707724 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Mon, 13 Nov 2023 23:36:32 -0800 Subject: [PATCH 25/72] Revert "Add missing newline" This reverts commit 9f90e435e43daf74ebd04ff48daa544a4afbaa80. --- source/adapters/hip/virtual_mem.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/adapters/hip/virtual_mem.cpp b/source/adapters/hip/virtual_mem.cpp index 6330451797..e2c4f9faf0 100644 --- a/source/adapters/hip/virtual_mem.cpp +++ b/source/adapters/hip/virtual_mem.cpp @@ -66,4 +66,4 @@ UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemGetInfo(ur_context_handle_t, detail::ur::die( "Virtual memory extension is not currently implemented for HIP adapter."); return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} +} \ No newline at end of file From dec5f32e3e368d6ad346b6d085ad7c768f0b9dd6 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Mon, 13 Nov 2023 23:36:45 -0800 Subject: [PATCH 26/72] Revert "Zero-initialize CUmemAccessDesc" This reverts commit 9ae8a992a9b40e04ba6b98b672b6f3e7e385e849. --- source/adapters/cuda/virtual_mem.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/adapters/cuda/virtual_mem.cpp b/source/adapters/cuda/virtual_mem.cpp index f49da3132e..203d052619 100644 --- a/source/adapters/cuda/virtual_mem.cpp +++ b/source/adapters/cuda/virtual_mem.cpp @@ -67,7 +67,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemFree( UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemSetAccess(ur_context_handle_t hContext, const void *pStart, size_t size, ur_virtual_mem_access_flags_t flags) { - CUmemAccessDesc AccessDesc = {}; + CUmemAccessDesc AccessDesc; if (flags & UR_VIRTUAL_MEM_ACCESS_FLAG_READ_WRITE) AccessDesc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE; else if (flags & UR_VIRTUAL_MEM_ACCESS_FLAG_READ_ONLY) From ca588860b7cb6c8f3b898f8e3d7356fb03ccd065 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Mon, 13 Nov 2023 23:37:02 -0800 Subject: [PATCH 27/72] Revert "Fix common include in cuda device.hpp" This reverts commit 6ba94ac426e3fb258f423be87e4a771f6db70060. --- source/adapters/cuda/device.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/adapters/cuda/device.hpp b/source/adapters/cuda/device.hpp index 08a1b5852a..5ad70672e9 100644 --- a/source/adapters/cuda/device.hpp +++ b/source/adapters/cuda/device.hpp @@ -11,8 +11,6 @@ #include -#include "common.hpp" - struct ur_device_handle_t_ { private: using native_type = CUdevice; From 11cca13bf4d6c14487a48ecb9b977cddb786456c Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Mon, 13 Nov 2023 23:37:19 -0800 Subject: [PATCH 28/72] Revert "Fix return of check error" This reverts commit c36bd72e3c5e94f0bc4756789d676a4d20b1dc43. --- source/adapters/cuda/virtual_mem.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/adapters/cuda/virtual_mem.cpp b/source/adapters/cuda/virtual_mem.cpp index 203d052619..583594fea1 100644 --- a/source/adapters/cuda/virtual_mem.cpp +++ b/source/adapters/cuda/virtual_mem.cpp @@ -52,8 +52,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemReserve(ur_context_handle_t hContext, const void *pStart, size_t size, void **ppStart) { ScopedContext Active(hContext); - UR_CHECK_ERROR(cuMemAddressReserve((CUdeviceptr *)ppStart, size, 0, - (CUdeviceptr)pStart, 0)); + return UR_CHECK_ERROR(cuMemAddressReserve((CUdeviceptr *)ppStart, size, 0, + (CUdeviceptr)pStart, 0)); return UR_RESULT_SUCCESS; } From 3e6194e59ba014b5d06aa00713eb3d07457d5bc3 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Mon, 13 Nov 2023 23:37:33 -0800 Subject: [PATCH 29/72] Revert "Remove duplicate CUDA UR_DEVICE_INFO_VIRTUAL_MEMORY_SUPPORT case" This reverts commit 1096b0f389e661b12a5c30c3edc5f84f09c9d32c. --- source/adapters/cuda/device.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/source/adapters/cuda/device.cpp b/source/adapters/cuda/device.cpp index 9677005d07..c27dbdbb52 100644 --- a/source/adapters/cuda/device.cpp +++ b/source/adapters/cuda/device.cpp @@ -1028,6 +1028,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, case UR_DEVICE_INFO_GPU_SUBSLICES_PER_SLICE: case UR_DEVICE_INFO_GPU_EU_COUNT_PER_SUBSLICE: case UR_DEVICE_INFO_GPU_HW_THREADS_PER_EU: + case UR_DEVICE_INFO_VIRTUAL_MEMORY_SUPPORT: return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; default: From 89febf0b3e580e6ec8a6a76a55724e7eafd752c1 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Mon, 13 Nov 2023 23:37:49 -0800 Subject: [PATCH 30/72] Revert "[UR][CUDA][L0][HIP] Add virtual memory adapter implementations" This reverts commit 67fcd5e11323ee2e2de88767951e5222661882f9. --- source/adapters/cuda/CMakeLists.txt | 3 - source/adapters/cuda/device.cpp | 2 - source/adapters/cuda/physical_mem.cpp | 58 -------- source/adapters/cuda/physical_mem.hpp | 68 --------- source/adapters/cuda/virtual_mem.cpp | 138 ------------------- source/adapters/hip/CMakeLists.txt | 3 - source/adapters/hip/device.cpp | 2 - source/adapters/hip/physical_mem.cpp | 36 ----- source/adapters/hip/physical_mem.hpp | 30 ---- source/adapters/hip/virtual_mem.cpp | 69 ---------- source/adapters/level_zero/CMakeLists.txt | 3 - source/adapters/level_zero/common.cpp | 3 - source/adapters/level_zero/device.cpp | 3 - source/adapters/level_zero/physical_mem.cpp | 54 -------- source/adapters/level_zero/physical_mem.hpp | 24 ---- source/adapters/level_zero/ur_level_zero.hpp | 1 - source/adapters/level_zero/virtual_mem.cpp | 120 ---------------- 17 files changed, 617 deletions(-) delete mode 100644 source/adapters/cuda/physical_mem.cpp delete mode 100644 source/adapters/cuda/physical_mem.hpp delete mode 100644 source/adapters/cuda/virtual_mem.cpp delete mode 100644 source/adapters/hip/physical_mem.cpp delete mode 100644 source/adapters/hip/physical_mem.hpp delete mode 100644 source/adapters/hip/virtual_mem.cpp delete mode 100644 source/adapters/level_zero/physical_mem.cpp delete mode 100644 source/adapters/level_zero/physical_mem.hpp delete mode 100644 source/adapters/level_zero/virtual_mem.cpp diff --git a/source/adapters/cuda/CMakeLists.txt b/source/adapters/cuda/CMakeLists.txt index cf97419035..4a8c1d1f59 100644 --- a/source/adapters/cuda/CMakeLists.txt +++ b/source/adapters/cuda/CMakeLists.txt @@ -27,8 +27,6 @@ add_ur_adapter(${TARGET_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/kernel.cpp ${CMAKE_CURRENT_SOURCE_DIR}/memory.hpp ${CMAKE_CURRENT_SOURCE_DIR}/memory.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.cpp ${CMAKE_CURRENT_SOURCE_DIR}/platform.hpp ${CMAKE_CURRENT_SOURCE_DIR}/platform.cpp ${CMAKE_CURRENT_SOURCE_DIR}/program.hpp @@ -40,7 +38,6 @@ add_ur_adapter(${TARGET_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/tracing.cpp ${CMAKE_CURRENT_SOURCE_DIR}/usm.cpp ${CMAKE_CURRENT_SOURCE_DIR}/usm_p2p.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/virtual_mem.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../ur/ur.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../ur/ur.hpp ) diff --git a/source/adapters/cuda/device.cpp b/source/adapters/cuda/device.cpp index c27dbdbb52..a4877236ae 100644 --- a/source/adapters/cuda/device.cpp +++ b/source/adapters/cuda/device.cpp @@ -1017,8 +1017,6 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, // TODO: Investigate if this information is available on CUDA. case UR_DEVICE_INFO_HOST_PIPE_READ_WRITE_SUPPORTED: return ReturnValue(false); - case UR_DEVICE_INFO_VIRTUAL_MEMORY_SUPPORT: - return ReturnValue(true); case UR_DEVICE_INFO_ESIMD_SUPPORT: return ReturnValue(false); case UR_DEVICE_INFO_MAX_READ_WRITE_IMAGE_ARGS: diff --git a/source/adapters/cuda/physical_mem.cpp b/source/adapters/cuda/physical_mem.cpp deleted file mode 100644 index 177bc2234d..0000000000 --- a/source/adapters/cuda/physical_mem.cpp +++ /dev/null @@ -1,58 +0,0 @@ -//===--------- physical_mem.cpp - CUDA Adapter ----------------------------===// -// -// Copyright (C) 2023 Intel Corporation -// -// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM -// Exceptions. See LICENSE.TXT -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "physical_mem.hpp" -#include "common.hpp" -#include "context.hpp" -#include "event.hpp" - -#include -#include - -UR_APIEXPORT ur_result_t UR_APICALL urPhysicalMemCreate( - ur_context_handle_t hContext, ur_device_handle_t hDevice, size_t size, - [[maybe_unused]] const ur_physical_mem_properties_t *pProperties, - ur_physical_mem_handle_t *phPhysicalMem) { - CUmemAllocationProp AllocProps = {}; - AllocProps.location.type = CU_MEM_LOCATION_TYPE_DEVICE; - AllocProps.type = CU_MEM_ALLOCATION_TYPE_PINNED; - UR_ASSERT(GetDeviceOrdinal(hDevice, AllocProps.location.id), - UR_RESULT_ERROR_INVALID_DEVICE); - - CUmemGenericAllocationHandle ResHandle; - UR_CHECK_ERROR(cuMemCreate(&ResHandle, size, &AllocProps, 0)); - *phPhysicalMem = new ur_physical_mem_handle_t_(ResHandle, hContext); - - return UR_RESULT_SUCCESS; -} - -UR_APIEXPORT ur_result_t UR_APICALL -urPhysicalMemRetain(ur_physical_mem_handle_t hPhysicalMem) { - hPhysicalMem->incrementReferenceCount(); - return UR_RESULT_SUCCESS; -} - -UR_APIEXPORT ur_result_t UR_APICALL -urPhysicalMemRelease(ur_physical_mem_handle_t hPhysicalMem) { - if (hPhysicalMem->decrementReferenceCount() > 0) - return UR_RESULT_SUCCESS; - - try { - std::unique_ptr PhysicalMemGuard(hPhysicalMem); - - ScopedContext Active(hPhysicalMem->getContext()); - UR_CHECK_ERROR(cuMemRelease(hPhysicalMem->get())); - return UR_RESULT_SUCCESS; - } catch (ur_result_t err) { - return err; - } catch (...) { - return UR_RESULT_ERROR_OUT_OF_RESOURCES; - } -} diff --git a/source/adapters/cuda/physical_mem.hpp b/source/adapters/cuda/physical_mem.hpp deleted file mode 100644 index 2b0dc029d5..0000000000 --- a/source/adapters/cuda/physical_mem.hpp +++ /dev/null @@ -1,68 +0,0 @@ -//===---------- physical_mem.hpp - CUDA Adapter ---------------------------===// -// -// Copyright (C) 2023 Intel Corporation -// -// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM -// Exceptions. See LICENSE.TXT -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -#pragma once - -#include - -#include - -#include "adapter.hpp" -#include "device.hpp" -#include "platform.hpp" - -/// UR queue mapping on physical memory allocations used in virtual memory -/// management. -/// -struct ur_physical_mem_handle_t_ { - using native_type = CUmemGenericAllocationHandle; - - std::atomic_uint32_t RefCount; - native_type PhysicalMem; - ur_context_handle_t_ *Context; - - ur_physical_mem_handle_t_(native_type PhysMem, ur_context_handle_t_ *Ctx) - : RefCount(1), PhysicalMem(PhysMem), Context(Ctx) { - urContextRetain(Context); - } - - ~ur_physical_mem_handle_t_() { urContextRelease(Context); } - - native_type get() const noexcept { return PhysicalMem; } - - ur_context_handle_t_ *getContext() const noexcept { return Context; } - - uint32_t incrementReferenceCount() noexcept { return ++RefCount; } - - uint32_t decrementReferenceCount() noexcept { return --RefCount; } - - uint32_t getReferenceCount() const noexcept { return RefCount; } -}; - -// Find a device ordinal of a device. -inline ur_result_t GetDeviceOrdinal(ur_device_handle_t Device, int &Ordinal) { - ur_adapter_handle_t AdapterHandle = &adapter; - // Get list of platforms - uint32_t NumPlatforms; - UR_ASSERT(urPlatformGet(&AdapterHandle, 1, 0, nullptr, &NumPlatforms), - UR_RESULT_ERROR_INVALID_ARGUMENT); - UR_ASSERT(NumPlatforms, UR_RESULT_ERROR_UNKNOWN); - - std::vector Platforms{NumPlatforms}; - UR_ASSERT( - urPlatformGet(&AdapterHandle, 1, NumPlatforms, Platforms.data(), nullptr), - UR_RESULT_ERROR_INVALID_ARGUMENT); - - // Ordinal corresponds to the platform ID as each device has its own platform. - CUdevice NativeDevice = Device->get(); - for (Ordinal = 0; size_t(Ordinal) < Platforms.size(); ++Ordinal) - if (Platforms[Ordinal]->Devices[0]->get() == NativeDevice) - return UR_RESULT_SUCCESS; - return UR_RESULT_ERROR_INVALID_DEVICE; -} diff --git a/source/adapters/cuda/virtual_mem.cpp b/source/adapters/cuda/virtual_mem.cpp deleted file mode 100644 index 583594fea1..0000000000 --- a/source/adapters/cuda/virtual_mem.cpp +++ /dev/null @@ -1,138 +0,0 @@ -//===--------- virtual_mem.cpp - CUDA Adapter -----------------------------===// -// -// Copyright (C) 2023 Intel Corporation -// -// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM -// Exceptions. See LICENSE.TXT -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "common.hpp" -#include "context.hpp" -#include "event.hpp" -#include "physical_mem.hpp" - -#include -#include - -UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemGranularityGetInfo( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - ur_virtual_mem_granularity_info_t propName, size_t propSize, - void *pPropValue, size_t *pPropSizeRet) { - UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); - - ScopedContext Active(hContext); - switch (propName) { - case UR_VIRTUAL_MEM_GRANULARITY_INFO_MINIMUM: - case UR_VIRTUAL_MEM_GRANULARITY_INFO_RECOMMENDED: { - CUmemAllocationGranularity_flags Flags = - propName == UR_VIRTUAL_MEM_GRANULARITY_INFO_MINIMUM - ? CU_MEM_ALLOC_GRANULARITY_MINIMUM - : CU_MEM_ALLOC_GRANULARITY_RECOMMENDED; - CUmemAllocationProp AllocProps = {}; - AllocProps.location.type = CU_MEM_LOCATION_TYPE_DEVICE; - AllocProps.type = CU_MEM_ALLOCATION_TYPE_PINNED; - UR_ASSERT(GetDeviceOrdinal(hDevice, AllocProps.location.id), - UR_RESULT_ERROR_INVALID_DEVICE); - - size_t Granularity; - UR_CHECK_ERROR( - cuMemGetAllocationGranularity(&Granularity, &AllocProps, Flags)); - return ReturnValue(Granularity); - } - default: - return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; - } - - return UR_RESULT_SUCCESS; -} - -UR_APIEXPORT ur_result_t UR_APICALL -urVirtualMemReserve(ur_context_handle_t hContext, const void *pStart, - size_t size, void **ppStart) { - ScopedContext Active(hContext); - return UR_CHECK_ERROR(cuMemAddressReserve((CUdeviceptr *)ppStart, size, 0, - (CUdeviceptr)pStart, 0)); - return UR_RESULT_SUCCESS; -} - -UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemFree( - ur_context_handle_t hContext, const void *pStart, size_t size) { - ScopedContext Active(hContext); - UR_CHECK_ERROR(cuMemAddressFree((CUdeviceptr)pStart, size)); - return UR_RESULT_SUCCESS; -} - -UR_APIEXPORT ur_result_t UR_APICALL -urVirtualMemSetAccess(ur_context_handle_t hContext, const void *pStart, - size_t size, ur_virtual_mem_access_flags_t flags) { - CUmemAccessDesc AccessDesc; - if (flags & UR_VIRTUAL_MEM_ACCESS_FLAG_READ_WRITE) - AccessDesc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE; - else if (flags & UR_VIRTUAL_MEM_ACCESS_FLAG_READ_ONLY) - AccessDesc.flags = CU_MEM_ACCESS_FLAGS_PROT_READ; - else - AccessDesc.flags = CU_MEM_ACCESS_FLAGS_PROT_NONE; - AccessDesc.location.type = CU_MEM_LOCATION_TYPE_DEVICE; - // TODO: When contexts support multiple devices, we should create a descriptor - // for each. We may also introduce a variant of this function with a - // specific device. - UR_ASSERT(GetDeviceOrdinal(hContext->getDevice(), AccessDesc.location.id), - UR_RESULT_ERROR_INVALID_DEVICE); - - ScopedContext Active(hContext); - UR_CHECK_ERROR(cuMemSetAccess((CUdeviceptr)pStart, size, &AccessDesc, 1)); - return UR_RESULT_SUCCESS; -} - -UR_APIEXPORT ur_result_t UR_APICALL -urVirtualMemMap(ur_context_handle_t hContext, const void *pStart, size_t size, - ur_physical_mem_handle_t hPhysicalMem, size_t offset, - ur_virtual_mem_access_flags_t flags) { - ScopedContext Active(hContext); - UR_CHECK_ERROR( - cuMemMap((CUdeviceptr)pStart, size, offset, hPhysicalMem->get(), 0)); - if (flags) - UR_ASSERT(urVirtualMemSetAccess(hContext, pStart, size, flags), - UR_RESULT_ERROR_INVALID_ARGUMENT); - return UR_RESULT_SUCCESS; -} - -UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemUnmap( - ur_context_handle_t hContext, const void *pStart, size_t size) { - ScopedContext Active(hContext); - UR_CHECK_ERROR(cuMemUnmap((CUdeviceptr)pStart, size)); - return UR_RESULT_SUCCESS; -} - -UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemGetInfo( - ur_context_handle_t hContext, const void *pStart, - [[maybe_unused]] size_t size, ur_virtual_mem_info_t propName, - size_t propSize, void *pPropValue, size_t *pPropSizeRet) { - UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); - - ScopedContext Active(hContext); - switch (propName) { - case UR_VIRTUAL_MEM_INFO_ACCESS_MODE: { - CUmemLocation MemLocation = {}; - MemLocation.type = CU_MEM_LOCATION_TYPE_DEVICE; - UR_ASSERT(GetDeviceOrdinal(hContext->getDevice(), MemLocation.id), - UR_RESULT_ERROR_INVALID_DEVICE); - - unsigned long long CuAccessFlags; - UR_CHECK_ERROR( - cuMemGetAccess(&CuAccessFlags, &MemLocation, (CUdeviceptr)pStart)); - - ur_virtual_mem_access_flags_t UrAccessFlags = 0; - if (CuAccessFlags == CU_MEM_ACCESS_FLAGS_PROT_READWRITE) - UrAccessFlags = UR_VIRTUAL_MEM_ACCESS_FLAG_READ_WRITE; - else if (CuAccessFlags == CU_MEM_ACCESS_FLAGS_PROT_READ) - UrAccessFlags = UR_VIRTUAL_MEM_ACCESS_FLAG_READ_ONLY; - return ReturnValue(UrAccessFlags); - } - default: - return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; - } - return UR_RESULT_SUCCESS; -} diff --git a/source/adapters/hip/CMakeLists.txt b/source/adapters/hip/CMakeLists.txt index 1ed9d52c2b..7de1b5f501 100644 --- a/source/adapters/hip/CMakeLists.txt +++ b/source/adapters/hip/CMakeLists.txt @@ -61,8 +61,6 @@ add_ur_adapter(${TARGET_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/kernel.cpp ${CMAKE_CURRENT_SOURCE_DIR}/memory.hpp ${CMAKE_CURRENT_SOURCE_DIR}/memory.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.cpp ${CMAKE_CURRENT_SOURCE_DIR}/platform.hpp ${CMAKE_CURRENT_SOURCE_DIR}/platform.cpp ${CMAKE_CURRENT_SOURCE_DIR}/program.hpp @@ -73,7 +71,6 @@ add_ur_adapter(${TARGET_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/sampler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/usm.cpp ${CMAKE_CURRENT_SOURCE_DIR}/usm_p2p.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/virtual_mem.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../ur/ur.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../ur/ur.hpp ) diff --git a/source/adapters/hip/device.cpp b/source/adapters/hip/device.cpp index d199e90757..7cec6def8b 100644 --- a/source/adapters/hip/device.cpp +++ b/source/adapters/hip/device.cpp @@ -809,8 +809,6 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_HOST_PIPE_READ_WRITE_SUPPORTED: return ReturnValue(false); - case UR_DEVICE_INFO_VIRTUAL_MEMORY_SUPPORT: - return ReturnValue(false); case UR_DEVICE_INFO_ESIMD_SUPPORT: return ReturnValue(false); diff --git a/source/adapters/hip/physical_mem.cpp b/source/adapters/hip/physical_mem.cpp deleted file mode 100644 index 8939d89d33..0000000000 --- a/source/adapters/hip/physical_mem.cpp +++ /dev/null @@ -1,36 +0,0 @@ -//===--------- physical_mem.cpp - HIP Adapter -----------------------------===// -// -// Copyright (C) 2023 Intel Corporation -// -// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM -// Exceptions. See LICENSE.TXT -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "physical_mem.hpp" -#include "common.hpp" -#include "context.hpp" -#include "event.hpp" - -UR_APIEXPORT ur_result_t UR_APICALL urPhysicalMemCreate( - ur_context_handle_t, ur_device_handle_t, size_t, - const ur_physical_mem_properties_t *, ur_physical_mem_handle_t *) { - detail::ur::die( - "Virtual memory extension is not currently implemented for HIP adapter."); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - -UR_APIEXPORT ur_result_t UR_APICALL -urPhysicalMemRetain(ur_physical_mem_handle_t) { - detail::ur::die( - "Virtual memory extension is not currently implemented for HIP adapter."); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - -UR_APIEXPORT ur_result_t UR_APICALL -urPhysicalMemRelease(ur_physical_mem_handle_t) { - detail::ur::die( - "Virtual memory extension is not currently implemented for HIP adapter."); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} diff --git a/source/adapters/hip/physical_mem.hpp b/source/adapters/hip/physical_mem.hpp deleted file mode 100644 index fc50836f62..0000000000 --- a/source/adapters/hip/physical_mem.hpp +++ /dev/null @@ -1,30 +0,0 @@ -//===---------- physical_mem.hpp - HIP Adapter ----------------------------===// -// -// Copyright (C) 2023 Intel Corporation -// -// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM -// Exceptions. See LICENSE.TXT -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -#pragma once - -#include "common.hpp" -#include "device.hpp" -#include "platform.hpp" - -/// UR queue mapping on physical memory allocations used in virtual memory -/// management. -/// TODO: Implement. -/// -struct ur_physical_mem_handle_t_ { - std::atomic_uint32_t RefCount; - - ur_physical_mem_handle_t_() : RefCount(1) {} - - uint32_t incrementReferenceCount() noexcept { return ++RefCount; } - - uint32_t decrementReferenceCount() noexcept { return --RefCount; } - - uint32_t getReferenceCount() const noexcept { return RefCount; } -}; diff --git a/source/adapters/hip/virtual_mem.cpp b/source/adapters/hip/virtual_mem.cpp deleted file mode 100644 index e2c4f9faf0..0000000000 --- a/source/adapters/hip/virtual_mem.cpp +++ /dev/null @@ -1,69 +0,0 @@ -//===--------- virtual_mem.cpp - HIP Adapter ------------------------------===// -// -// Copyright (C) 2023 Intel Corporation -// -// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM -// Exceptions. See LICENSE.TXT -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "common.hpp" -#include "context.hpp" -#include "event.hpp" -#include "physical_mem.hpp" - -UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemGranularityGetInfo( - ur_context_handle_t, ur_device_handle_t, ur_virtual_mem_granularity_info_t, - size_t, void *, size_t *) { - detail::ur::die( - "Virtual memory extension is not currently implemented for HIP adapter."); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - -UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemReserve(ur_context_handle_t, - const void *, size_t, - void **) { - detail::ur::die( - "Virtual memory extension is not currently implemented for HIP adapter."); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - -UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemFree(ur_context_handle_t, - const void *, size_t) { - detail::ur::die( - "Virtual memory extension is not currently implemented for HIP adapter."); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - -UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemSetAccess( - ur_context_handle_t, const void *, size_t, ur_virtual_mem_access_flags_t) { - detail::ur::die( - "Virtual memory extension is not currently implemented for HIP adapter."); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - -UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemMap( - ur_context_handle_t, const void *, size_t, ur_physical_mem_handle_t, size_t, - ur_virtual_mem_access_flags_t) { - detail::ur::die( - "Virtual memory extension is not currently implemented for HIP adapter."); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - -UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemUnmap(ur_context_handle_t, - const void *, size_t) { - detail::ur::die( - "Virtual memory extension is not currently implemented for HIP adapter."); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - -UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemGetInfo(ur_context_handle_t, - const void *, size_t, - ur_virtual_mem_info_t, - size_t, void *, - size_t *) { - detail::ur::die( - "Virtual memory extension is not currently implemented for HIP adapter."); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} \ No newline at end of file diff --git a/source/adapters/level_zero/CMakeLists.txt b/source/adapters/level_zero/CMakeLists.txt index b80c5aef5d..223692e109 100644 --- a/source/adapters/level_zero/CMakeLists.txt +++ b/source/adapters/level_zero/CMakeLists.txt @@ -84,7 +84,6 @@ add_ur_adapter(${TARGET_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/usm.hpp ${CMAKE_CURRENT_SOURCE_DIR}/memory.hpp ${CMAKE_CURRENT_SOURCE_DIR}/kernel.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.hpp ${CMAKE_CURRENT_SOURCE_DIR}/platform.hpp ${CMAKE_CURRENT_SOURCE_DIR}/program.hpp ${CMAKE_CURRENT_SOURCE_DIR}/queue.hpp @@ -96,10 +95,8 @@ add_ur_adapter(${TARGET_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/event.cpp ${CMAKE_CURRENT_SOURCE_DIR}/usm.cpp ${CMAKE_CURRENT_SOURCE_DIR}/usm_p2p.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/virtual_mem.cpp ${CMAKE_CURRENT_SOURCE_DIR}/memory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/kernel.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.cpp ${CMAKE_CURRENT_SOURCE_DIR}/platform.cpp ${CMAKE_CURRENT_SOURCE_DIR}/program.cpp ${CMAKE_CURRENT_SOURCE_DIR}/queue.cpp diff --git a/source/adapters/level_zero/common.cpp b/source/adapters/level_zero/common.cpp index 47f0e46c91..eb0f34307c 100644 --- a/source/adapters/level_zero/common.cpp +++ b/source/adapters/level_zero/common.cpp @@ -208,9 +208,6 @@ template <> ze_structure_type_t getZeStructureType() { template <> ze_structure_type_t getZeStructureType() { return ZE_STRUCTURE_TYPE_SAMPLER_DESC; } -template <> ze_structure_type_t getZeStructureType() { - return ZE_STRUCTURE_TYPE_PHYSICAL_MEM_DESC; -} template <> ze_structure_type_t getZeStructureType() { return ZE_STRUCTURE_TYPE_DRIVER_PROPERTIES; } diff --git a/source/adapters/level_zero/device.cpp b/source/adapters/level_zero/device.cpp index fd085f1973..35e48931b2 100644 --- a/source/adapters/level_zero/device.cpp +++ b/source/adapters/level_zero/device.cpp @@ -790,9 +790,6 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo( return ReturnValue(static_cast( 0)); //__read_write attribute currently undefinde in opencl } - case UR_DEVICE_INFO_VIRTUAL_MEMORY_SUPPORT: { - return ReturnValue(static_cast(true)); - } case UR_DEVICE_INFO_ESIMD_SUPPORT: { // ESIMD is only supported by Intel GPUs. diff --git a/source/adapters/level_zero/physical_mem.cpp b/source/adapters/level_zero/physical_mem.cpp deleted file mode 100644 index d4d9792f24..0000000000 --- a/source/adapters/level_zero/physical_mem.cpp +++ /dev/null @@ -1,54 +0,0 @@ -//===---------------- physical_mem.cpp - Level Zero Adapter ---------------===// -// -// Copyright (C) 2023 Intel Corporation -// -// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM -// Exceptions. See LICENSE.TXT -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "physical_mem.hpp" -#include "common.hpp" -#include "context.hpp" -#include "device.hpp" -#include "ur_level_zero.hpp" - -UR_APIEXPORT ur_result_t UR_APICALL urPhysicalMemCreate( - ur_context_handle_t hContext, ur_device_handle_t hDevice, size_t size, - [[maybe_unused]] const ur_physical_mem_properties_t *pProperties, - ur_physical_mem_handle_t *phPhysicalMem) { - ZeStruct PhysicalMemDesc; - PhysicalMemDesc.flags = 0; - PhysicalMemDesc.size = size; - - ze_physical_mem_handle_t ZePhysicalMem; - ZE2UR_CALL(zePhysicalMemCreate, (hContext->ZeContext, hDevice->ZeDevice, - &PhysicalMemDesc, &ZePhysicalMem)); - try { - *phPhysicalMem = new ur_physical_mem_handle_t_(ZePhysicalMem, hContext); - } catch (const std::bad_alloc &) { - return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } catch (...) { - return UR_RESULT_ERROR_UNKNOWN; - } - return UR_RESULT_SUCCESS; -} - -UR_APIEXPORT ur_result_t UR_APICALL -urPhysicalMemRetain(ur_physical_mem_handle_t hPhysicalMem) { - hPhysicalMem->RefCount.increment(); - return UR_RESULT_SUCCESS; -} - -UR_APIEXPORT ur_result_t UR_APICALL -urPhysicalMemRelease(ur_physical_mem_handle_t hPhysicalMem) { - if (!hPhysicalMem->RefCount.decrementAndTest()) - return UR_RESULT_SUCCESS; - - ZE2UR_CALL(zePhysicalMemDestroy, - (hPhysicalMem->Context->ZeContext, hPhysicalMem->ZePhysicalMem)); - delete hPhysicalMem; - - return UR_RESULT_SUCCESS; -} diff --git a/source/adapters/level_zero/physical_mem.hpp b/source/adapters/level_zero/physical_mem.hpp deleted file mode 100644 index 9b83d93222..0000000000 --- a/source/adapters/level_zero/physical_mem.hpp +++ /dev/null @@ -1,24 +0,0 @@ -//===---------------- physical_mem.hpp - Level Zero Adapter ---------------===// -// -// Copyright (C) 2023 Intel Corporation -// -// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM -// Exceptions. See LICENSE.TXT -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -#pragma once - -#include "common.hpp" - -struct ur_physical_mem_handle_t_ : _ur_object { - ur_physical_mem_handle_t_(ze_physical_mem_handle_t ZePhysicalMem, - ur_context_handle_t Context) - : ZePhysicalMem{ZePhysicalMem}, Context{Context} {} - - // Level Zero physical memory handle. - ze_physical_mem_handle_t ZePhysicalMem; - - // Keeps the PI context of this memory handle. - ur_context_handle_t Context; -}; diff --git a/source/adapters/level_zero/ur_level_zero.hpp b/source/adapters/level_zero/ur_level_zero.hpp index dd7bbf67b3..38bc5aed2d 100644 --- a/source/adapters/level_zero/ur_level_zero.hpp +++ b/source/adapters/level_zero/ur_level_zero.hpp @@ -30,7 +30,6 @@ #include "image.hpp" #include "kernel.hpp" #include "memory.hpp" -#include "physical_mem.hpp" #include "platform.hpp" #include "program.hpp" #include "queue.hpp" diff --git a/source/adapters/level_zero/virtual_mem.cpp b/source/adapters/level_zero/virtual_mem.cpp deleted file mode 100644 index e90aec45de..0000000000 --- a/source/adapters/level_zero/virtual_mem.cpp +++ /dev/null @@ -1,120 +0,0 @@ -//===---------------- virtual_mem.cpp - Level Zero Adapter ----------------===// -// -// Copyright (C) 2023 Intel Corporation -// -// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM -// Exceptions. See LICENSE.TXT -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "common.hpp" -#include "context.hpp" -#include "device.hpp" -#include "physical_mem.hpp" -#include "ur_level_zero.hpp" - -UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemGranularityGetInfo( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - ur_virtual_mem_granularity_info_t propName, size_t propSize, - void *pPropValue, size_t *pPropSizeRet) { - UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); - switch (propName) { - case UR_VIRTUAL_MEM_GRANULARITY_INFO_MINIMUM: - case UR_VIRTUAL_MEM_GRANULARITY_INFO_RECOMMENDED: { - // For L0 the minimum and recommended granularity is the same. We use an - // memory size of 1 byte to get the actual granularity instead of the - // aligned size. - size_t PageSize; - ZE2UR_CALL(zeVirtualMemQueryPageSize, - (hContext->ZeContext, hDevice->ZeDevice, 1, &PageSize)); - return ReturnValue(PageSize); - } - default: - urPrint("Unsupported propName in urQueueGetInfo: propName=%d(0x%x)\n", - propName, propName); - return UR_RESULT_ERROR_INVALID_VALUE; - } - return UR_RESULT_SUCCESS; -} - -UR_APIEXPORT ur_result_t UR_APICALL -urVirtualMemReserve(ur_context_handle_t hContext, const void *pStart, - size_t size, void **ppStart) { - ZE2UR_CALL(zeVirtualMemReserve, (hContext->ZeContext, pStart, size, ppStart)); - - return UR_RESULT_SUCCESS; -} - -UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemFree( - ur_context_handle_t hContext, const void *pStart, size_t size) { - ZE2UR_CALL(zeVirtualMemFree, (hContext->ZeContext, pStart, size)); - - return UR_RESULT_SUCCESS; -} - -UR_APIEXPORT ur_result_t UR_APICALL -urVirtualMemSetAccess(ur_context_handle_t hContext, const void *pStart, - size_t size, ur_virtual_mem_access_flags_t flags) { - ze_memory_access_attribute_t AccessAttr = ZE_MEMORY_ACCESS_ATTRIBUTE_NONE; - if (flags & UR_VIRTUAL_MEM_ACCESS_FLAG_READ_WRITE) - AccessAttr = ZE_MEMORY_ACCESS_ATTRIBUTE_READWRITE; - else if (flags & UR_VIRTUAL_MEM_ACCESS_FLAG_READ_ONLY) - AccessAttr = ZE_MEMORY_ACCESS_ATTRIBUTE_READONLY; - - ZE2UR_CALL(zeVirtualMemSetAccessAttribute, - (hContext->ZeContext, pStart, size, AccessAttr)); - - return UR_RESULT_SUCCESS; -} - -UR_APIEXPORT ur_result_t UR_APICALL -urVirtualMemMap(ur_context_handle_t hContext, const void *pStart, size_t size, - ur_physical_mem_handle_t hPhysicalMem, size_t offset, - ur_virtual_mem_access_flags_t flags) { - ze_memory_access_attribute_t AccessAttr = ZE_MEMORY_ACCESS_ATTRIBUTE_NONE; - if (flags & UR_VIRTUAL_MEM_ACCESS_FLAG_READ_WRITE) - AccessAttr = ZE_MEMORY_ACCESS_ATTRIBUTE_READWRITE; - else if (flags & UR_VIRTUAL_MEM_ACCESS_FLAG_READ_ONLY) - AccessAttr = ZE_MEMORY_ACCESS_ATTRIBUTE_READONLY; - - ZE2UR_CALL(zeVirtualMemMap, - (hContext->ZeContext, pStart, size, hPhysicalMem->ZePhysicalMem, - offset, AccessAttr)); - - return UR_RESULT_SUCCESS; -} - -UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemUnmap( - ur_context_handle_t hContext, const void *pStart, size_t size) { - ZE2UR_CALL(zeVirtualMemUnmap, (hContext->ZeContext, pStart, size)); - - return UR_RESULT_SUCCESS; -} - -UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemGetInfo( - ur_context_handle_t hContext, const void *pStart, - [[maybe_unused]] size_t size, ur_virtual_mem_info_t propName, - size_t propSize, void *pPropValue, size_t *pPropSizeRet) { - UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); - switch (propName) { - case UR_VIRTUAL_MEM_INFO_ACCESS_MODE: { - size_t QuerySize; - ze_memory_access_attribute_t Access; - ZE2UR_CALL(zeVirtualMemGetAccessAttribute, - (hContext->ZeContext, pStart, size, &Access, &QuerySize)); - ur_virtual_mem_access_flags_t RetFlags = 0; - if (Access & ZE_MEMORY_ACCESS_ATTRIBUTE_READWRITE) - RetFlags |= UR_VIRTUAL_MEM_ACCESS_FLAG_READ_WRITE; - if (Access & ZE_MEMORY_ACCESS_ATTRIBUTE_READONLY) - RetFlags |= UR_VIRTUAL_MEM_ACCESS_FLAG_READ_ONLY; - return ReturnValue(Access); - } - default: - urPrint("Unsupported propName in urQueueGetInfo: propName=%d(0x%x)\n", - propName, propName); - return UR_RESULT_ERROR_INVALID_VALUE; - } - - return UR_RESULT_SUCCESS; -} From 1b81df9675c5b7cc6e5fd3c0940700bbf5da6021 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Mon, 13 Nov 2023 23:48:28 -0800 Subject: [PATCH 31/72] UR: revert gitignore --- .gitignore | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitignore b/.gitignore index 2166acc643..85770fe15c 100644 --- a/.gitignore +++ b/.gitignore @@ -86,5 +86,3 @@ out/ # External content */**/external - -.cache From 030400d74a9b1d7e30272e21fd3ef9b187086186 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Mon, 13 Nov 2023 23:49:50 -0800 Subject: [PATCH 32/72] UR: cmake option --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8ef4ef9a08..f1e9e736d5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,7 +33,7 @@ option(UR_USE_UBSAN "enable UndefinedBehaviorSanitizer" OFF) option(UR_USE_MSAN "enable MemorySanitizer" OFF) option(UR_USE_TSAN "enable ThreadSanitizer" OFF) option(UR_ENABLE_TRACING "enable api tracing through xpti" OFF) -option(UR_ENABLE_SANITIZER "enable device sanitizer" ON) +option(UR_ENABLE_SANITIZER "enable device sanitizer" OFF) option(UMF_BUILD_SHARED_LIBRARY "Build UMF as shared library" OFF) option(UMF_ENABLE_POOL_TRACKING "Build UMF with pool tracking" OFF) option(UR_BUILD_ADAPTER_L0 "build level 0 adapter from SYCL" OFF) From 7208518af06efe82ea476cd5ffce25e9d5e2f4cb Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Mon, 13 Nov 2023 23:52:36 -0800 Subject: [PATCH 33/72] UR: clean code --- source/adapters/level_zero/ur_interface_loader.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/adapters/level_zero/ur_interface_loader.cpp b/source/adapters/level_zero/ur_interface_loader.cpp index 439deb055c..5f6da8fd86 100644 --- a/source/adapters/level_zero/ur_interface_loader.cpp +++ b/source/adapters/level_zero/ur_interface_loader.cpp @@ -405,9 +405,9 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetVirtualMemProcAddrTable( pDdiTable->pfnFree = nullptr; pDdiTable->pfnGetInfo = nullptr; - pDdiTable->pfnGranularityGetInfo = urVirtualMemGranularityGetInfo; - pDdiTable->pfnMap = urVirtualMemMap; - pDdiTable->pfnReserve = urVirtualMemReserve; + pDdiTable->pfnGranularityGetInfo = nullptr; + pDdiTable->pfnMap = nullptr; + pDdiTable->pfnReserve = nullptr; pDdiTable->pfnSetAccess = nullptr; pDdiTable->pfnUnmap = nullptr; @@ -424,7 +424,7 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetPhysicalMemProcAddrTable( return retVal; } - pDdiTable->pfnCreate = urPhysicalMemCreate; + pDdiTable->pfnCreate = nullptr; pDdiTable->pfnRelease = nullptr; pDdiTable->pfnRetain = nullptr; From 668a11e8dd2018583ffe6e304688c706d91a608c Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Tue, 14 Nov 2023 00:01:41 -0800 Subject: [PATCH 34/72] UR: clean code --- .../layers/sanitizer/asan_interceptor.cpp | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index 38e10cc257..70a049dc2d 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -36,34 +36,6 @@ const int kUsmHostRedzoneMagic = 0x82; const int kUsmSharedRedzoneMagic = 0x83; const int kMemBufferRedzoneMagic = 0x84; -const int kUsmDeviceDeallocatedMagic = 0x91; -const int kUsmHostDeallocatedMagic = 0x92; -const int kUsmSharedDeallocatedMagic = 0x93; - -// Same with Asan Stack -const int kPrivateLeftRedzoneMagic = 0xf1; -const int kPrivateMidRedzoneMagic = 0xf2; -const int kPrivateRightRedzoneMagic = 0xf3; - -// These magic values are written to shadow for better error -// reporting. -// const int kAsanHeapLeftRedzoneMagic = 0xfa; -// const int kAsanHeapFreeMagic = 0xfd; -// const int kAsanStackLeftRedzoneMagic = 0xf1; -// const int kAsanStackMidRedzoneMagic = 0xf2; -// const int kAsanStackRightRedzoneMagic = 0xf3; -// const int kAsanStackAfterReturnMagic = 0xf5; -// const int kAsanInitializationOrderMagic = 0xf6; -// const int kAsanUserPoisonedMemoryMagic = 0xf7; -// const int kAsanContiguousContainerOOBMagic = 0xfc; -// const int kAsanStackUseAfterScopeMagic = 0xf8; -// const int kAsanGlobalRedzoneMagic = 0xf9; -// const int kAsanInternalHeapMagic = 0xfe; -// const int kAsanArrayCookieMagic = 0xac; -// const int kAsanIntraObjectRedzone = 0xbb; -// const int kAsanAllocaLeftMagic = 0xca; -// const int kAsanAllocaRightMagic = 0xcb; - const auto kSPIR_AsanShadowMemoryGlobalStart = "__AsanShadowMemoryGlobalStart"; const auto kSPIR_AsanShadowMemoryGlobalEnd = "__AsanShadowMemoryGlobalEnd"; From f623135fa928f6d9d1e659482cf560f59959db5d Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Tue, 14 Nov 2023 00:03:55 -0800 Subject: [PATCH 35/72] UR: clean code --- source/loader/layers/sanitizer/asan_interceptor.cpp | 6 +++--- source/loader/layers/sanitizer/asan_interceptor.hpp | 4 ++-- source/loader/layers/sanitizer/ur_sanddi.cpp | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index 70a049dc2d..48f6ac1f77 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -195,9 +195,9 @@ ur_result_t SanitizerInterceptor::releaseMemory(ur_context_handle_t Context, return m_Dditable.USM.pfnFree(Context, (void *)AllocInfo.AllocBegin); } -bool SanitizerInterceptor::launchKernel(ur_kernel_handle_t Kernel, - ur_queue_handle_t Queue, - ur_event_handle_t &Event) { +bool SanitizerInterceptor::preLaunchKernel(ur_kernel_handle_t Kernel, + ur_queue_handle_t Queue, + ur_event_handle_t &Event) { prepareLaunch(Queue, Kernel); updateShadowMemory(Queue); diff --git a/source/loader/layers/sanitizer/asan_interceptor.hpp b/source/loader/layers/sanitizer/asan_interceptor.hpp index de2ddd62ba..b5f6508345 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.hpp +++ b/source/loader/layers/sanitizer/asan_interceptor.hpp @@ -92,8 +92,8 @@ class SanitizerInterceptor { void **ResultPtr, USMMemoryType Type); ur_result_t releaseMemory(ur_context_handle_t Context, void *Ptr); - bool launchKernel(ur_kernel_handle_t Kernel, ur_queue_handle_t Queue, - ur_event_handle_t &Event); + bool preLaunchKernel(ur_kernel_handle_t Kernel, ur_queue_handle_t Queue, + ur_event_handle_t &Event); void postLaunchKernel(ur_kernel_handle_t Kernel, ur_queue_handle_t Queue, ur_event_handle_t *Event, bool SetCallback = true); diff --git a/source/loader/layers/sanitizer/ur_sanddi.cpp b/source/loader/layers/sanitizer/ur_sanddi.cpp index d75d0d891d..98c10155c4 100644 --- a/source/loader/layers/sanitizer/ur_sanddi.cpp +++ b/source/loader/layers/sanitizer/ur_sanddi.cpp @@ -166,7 +166,7 @@ __urdlllocal ur_result_t UR_APICALL urEnqueueKernelLaunch( } // launchKernel must append to num_events_in_wait_list, not prepend - context.interceptor->launchKernel(hKernel, hQueue, lk_event); + context.interceptor->preLaunchKernel(hKernel, hQueue, lk_event); if (lk_event) { events.push_back(lk_event); } From 7be86671bdf753d712b00799d37aa3afc9a408ef Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Tue, 14 Nov 2023 00:05:04 -0800 Subject: [PATCH 36/72] UR: clean code --- .../layers/sanitizer/asan_interceptor.cpp | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index 48f6ac1f77..8896865aef 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -43,9 +43,9 @@ const auto kSPIR_DeviceSanitizerReportMem = "__DeviceSanitizerReportMem"; DeviceSanitizerReport SPIR_DeviceSanitizerReportMem; -uptr MemToShadow_CPU(uptr USM_SHADOW_BASE, uptr UPtr) { - return USM_SHADOW_BASE + (UPtr >> 3); -} +// uptr MemToShadow_CPU(uptr USM_SHADOW_BASE, uptr UPtr) { +// return USM_SHADOW_BASE + (UPtr >> 3); +// } uptr MemToShadow_PVC(uptr USM_SHADOW_BASE, uptr UPtr) { if (UPtr & 0xFF00000000000000ULL) { // Device USM @@ -56,13 +56,13 @@ uptr MemToShadow_PVC(uptr USM_SHADOW_BASE, uptr UPtr) { } } -uptr MemToShadow_DG2(uptr USM_SHADOW_BASE, uptr UPtr) { - if (UPtr & (~0xFFFFFFFFFFFFULL)) { // Device USM - return USM_SHADOW_BASE + ((UPtr & 0xFFFFFFFFFFFFULL) >> 3); - } else { - return USM_SHADOW_BASE + (UPtr >> 3); - } -} +// uptr MemToShadow_DG2(uptr USM_SHADOW_BASE, uptr UPtr) { +// if (UPtr & (~0xFFFFFFFFFFFFULL)) { // Device USM +// return USM_SHADOW_BASE + ((UPtr & 0xFFFFFFFFFFFFULL) >> 3); +// } else { +// return USM_SHADOW_BASE + (UPtr >> 3); +// } +// } ur_context_handle_t getContext(ur_queue_handle_t Queue, ur_dditable_t &Dditable) { From e4a47ac2830e31572fdf66774d88ae7298cb9b2e Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Tue, 14 Nov 2023 00:21:30 -0800 Subject: [PATCH 37/72] UR: clean code --- .../layers/sanitizer/asan_interceptor.cpp | 122 ++++++++---------- .../layers/sanitizer/asan_interceptor.hpp | 13 +- .../loader/layers/sanitizer/ur_san_layer.cpp | 2 +- 3 files changed, 60 insertions(+), 77 deletions(-) diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index 8896865aef..24e9c434ad 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -64,31 +64,29 @@ uptr MemToShadow_PVC(uptr USM_SHADOW_BASE, uptr UPtr) { // } // } -ur_context_handle_t getContext(ur_queue_handle_t Queue, - ur_dditable_t &Dditable) { +ur_context_handle_t getContext(ur_queue_handle_t Queue) { ur_context_handle_t Context; - auto Result = Dditable.Queue.pfnGetInfo(Queue, UR_QUEUE_INFO_CONTEXT, - sizeof(ur_context_handle_t), - &Context, nullptr); + auto Result = context.urDdiTable.Queue.pfnGetInfo( + Queue, UR_QUEUE_INFO_CONTEXT, sizeof(ur_context_handle_t), &Context, + nullptr); assert(Result == UR_RESULT_SUCCESS); return Context; } -ur_device_handle_t getDevice(ur_queue_handle_t Queue, ur_dditable_t &Dditable) { +ur_device_handle_t getDevice(ur_queue_handle_t Queue) { ur_device_handle_t Device; - auto Result = - Dditable.Queue.pfnGetInfo(Queue, UR_QUEUE_INFO_DEVICE, - sizeof(ur_device_handle_t), &Device, nullptr); + auto Result = context.urDdiTable.Queue.pfnGetInfo( + Queue, UR_QUEUE_INFO_DEVICE, sizeof(ur_device_handle_t), &Device, + nullptr); assert(Result == UR_RESULT_SUCCESS); return Device; } -ur_program_handle_t getProgram(ur_kernel_handle_t Kernel, - ur_dditable_t &Dditable) { +ur_program_handle_t getProgram(ur_kernel_handle_t Kernel) { ur_program_handle_t Program; - auto Result = Dditable.Kernel.pfnGetInfo(Kernel, UR_KERNEL_INFO_PROGRAM, - sizeof(ur_program_handle_t), - &Program, nullptr); + auto Result = context.urDdiTable.Kernel.pfnGetInfo( + Kernel, UR_KERNEL_INFO_PROGRAM, sizeof(ur_program_handle_t), &Program, + nullptr); assert(Result == UR_RESULT_SUCCESS); return Program; } @@ -118,14 +116,14 @@ ur_result_t SanitizerInterceptor::allocateMemory( void *Allocated = nullptr; if (Type == USMMemoryType::DEVICE) { - UR_CALL(m_Dditable.USM.pfnDeviceAlloc(Context, Device, Properties, Pool, - NeededSize, &Allocated)); + UR_CALL(context.urDdiTable.USM.pfnDeviceAlloc( + Context, Device, Properties, Pool, NeededSize, &Allocated)); } else if (Type == USMMemoryType::HOST) { - UR_CALL(m_Dditable.USM.pfnHostAlloc(Context, Properties, Pool, - NeededSize, &Allocated)); + UR_CALL(context.urDdiTable.USM.pfnHostAlloc(Context, Properties, Pool, + NeededSize, &Allocated)); } else if (Type == USMMemoryType::SHARE) { - UR_CALL(m_Dditable.USM.pfnSharedAlloc(Context, Device, Properties, Pool, - NeededSize, &Allocated)); + UR_CALL(context.urDdiTable.USM.pfnSharedAlloc( + Context, Device, Properties, Pool, NeededSize, &Allocated)); } else { die("SanitizerInterceptor: unsupport memory type"); } @@ -192,7 +190,8 @@ ur_result_t SanitizerInterceptor::releaseMemory(ur_context_handle_t Context, } // TODO: Update shadow memory - return m_Dditable.USM.pfnFree(Context, (void *)AllocInfo.AllocBegin); + return context.urDdiTable.USM.pfnFree(Context, + (void *)AllocInfo.AllocBegin); } bool SanitizerInterceptor::preLaunchKernel(ur_kernel_handle_t Kernel, @@ -203,7 +202,7 @@ bool SanitizerInterceptor::preLaunchKernel(ur_kernel_handle_t Kernel, updateShadowMemory(Queue); // Return LastEvent in QueueInfo - auto Context = getContext(Queue, m_Dditable); + auto Context = getContext(Queue); auto &ContextInfo = getContextInfo(Context); auto &QueueInfo = ContextInfo.getQueueInfo(Queue); @@ -218,13 +217,13 @@ void SanitizerInterceptor::postLaunchKernel(ur_kernel_handle_t Kernel, ur_queue_handle_t Queue, ur_event_handle_t *Event, bool SetCallback) { - auto Program = getProgram(Kernel, m_Dditable); + auto Program = getProgram(Kernel); ur_event_handle_t ReadEvent{}; // If kernel has defined SPIR_DeviceSanitizerReportMem, then we try to read it // to host, but it's okay that it isn't defined - auto Result = m_Dditable.Enqueue.pfnDeviceGlobalVariableRead( + auto Result = context.urDdiTable.Enqueue.pfnDeviceGlobalVariableRead( Queue, Program, kSPIR_DeviceSanitizerReportMem, true, sizeof(SPIR_DeviceSanitizerReportMem), 0, &SPIR_DeviceSanitizerReportMem, 1, Event, &ReadEvent); @@ -258,14 +257,14 @@ void SanitizerInterceptor::postLaunchKernel(ur_kernel_handle_t Kernel, std::string SanitizerInterceptor::getKernelName(ur_kernel_handle_t Kernel) { size_t KernelNameSize = 0; - auto Res = m_Dditable.Kernel.pfnGetInfo( + auto Res = context.urDdiTable.Kernel.pfnGetInfo( Kernel, UR_KERNEL_INFO_FUNCTION_NAME, 0, nullptr, &KernelNameSize); assert(Res == UR_RESULT_SUCCESS); std::vector KernelNameBuf(KernelNameSize + 1); - Res = m_Dditable.Kernel.pfnGetInfo(Kernel, UR_KERNEL_INFO_FUNCTION_NAME, - KernelNameSize, KernelNameBuf.data(), - nullptr); + Res = context.urDdiTable.Kernel.pfnGetInfo( + Kernel, UR_KERNEL_INFO_FUNCTION_NAME, KernelNameSize, + KernelNameBuf.data(), nullptr); assert(Res == UR_RESULT_SUCCESS); KernelNameBuf[KernelNameSize] = '\0'; @@ -285,7 +284,7 @@ ur_result_t SanitizerInterceptor::allocShadowMemory(ur_context_handle_t Context, constexpr size_t SHADOW_SIZE = 1ULL << 46; // TODO: Protect Bad Zone - UR_CALL(m_Dditable.VirtualMem.pfnReserve( + UR_CALL(context.urDdiTable.VirtualMem.pfnReserve( Context, nullptr, SHADOW_SIZE, (void **)&DeviceInfo.ShadowOffset)); DeviceInfo.ShadowOffsetEnd = DeviceInfo.ShadowOffset + SHADOW_SIZE; @@ -298,24 +297,26 @@ ur_result_t SanitizerInterceptor::allocShadowMemory(ur_context_handle_t Context, return UR_RESULT_SUCCESS; } -ur_result_t SanitizerInterceptor::piextEnqueueMemSetShadow( +ur_result_t SanitizerInterceptor::enqueueMemSetShadow( ur_context_handle_t Context, ur_device_handle_t Device, - ur_queue_handle_t Queue, void *Ptr, size_t Size, uint8_t Value, - size_t NumEventsInWaitList, const ur_event_handle_t *EventsWaitList, - ur_event_handle_t *OutEvent) { + ur_queue_handle_t Queue, uptr Ptr, uptr Size, u8 Value, + ur_event_handle_t DepEvent, ur_event_handle_t *OutEvent) { + + uint32_t NumEventsInWaitList = DepEvent ? 1 : 0; + const ur_event_handle_t *EventsWaitList = DepEvent ? &DepEvent : nullptr; + auto &ContextInfo = getContextInfo(Context); auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); + if (DeviceInfo.Type == DeviceType::CPU) { die("Unsupport device type"); } else if (DeviceInfo.Type == DeviceType::GPU_PVC) { - const uptr UPtr = (uptr)Ptr; - ur_event_handle_t InternalEvent{}; ur_event_handle_t *Event = OutEvent ? OutEvent : &InternalEvent; - uptr ShadowBegin = MemToShadow_PVC(DeviceInfo.ShadowOffset, UPtr); + uptr ShadowBegin = MemToShadow_PVC(DeviceInfo.ShadowOffset, Ptr); uptr ShadowEnd = - MemToShadow_PVC(DeviceInfo.ShadowOffset, UPtr + Size - 1); + MemToShadow_PVC(DeviceInfo.ShadowOffset, Ptr + Size - 1); // Maybe in future, we needn't to map physical memory manually const bool IsNeedMapPhysicalMem = true; @@ -332,7 +333,7 @@ ur_result_t SanitizerInterceptor::piextEnqueueMemSetShadow( for (auto MappedPtr = RoundDownTo(ShadowBegin, PageSize); MappedPtr <= ShadowEnd; MappedPtr += PageSize) { if (!PhysicalMem) { - auto URes = m_Dditable.PhysicalMem.pfnCreate( + auto URes = context.urDdiTable.PhysicalMem.pfnCreate( Context, Device, PageSize, &Desc, &PhysicalMem); if (URes != UR_RESULT_SUCCESS) { context.logger.error("zePhysicalMemCreate(): {}", @@ -346,7 +347,7 @@ ur_result_t SanitizerInterceptor::piextEnqueueMemSetShadow( (void *)(MappedPtr + PageSize - 1)); // FIXME: No flag to check the failed reason is VA is already mapped - auto URes = m_Dditable.VirtualMem.pfnMap( + auto URes = context.urDdiTable.VirtualMem.pfnMap( Context, (void *)MappedPtr, PageSize, PhysicalMem, 0, UR_VIRTUAL_MEM_ACCESS_FLAG_READ_WRITE); if (URes != UR_RESULT_SUCCESS) { @@ -362,7 +363,7 @@ ur_result_t SanitizerInterceptor::piextEnqueueMemSetShadow( // FIXME: Maybe we needn't to initialize shadow memory to zero? Or it'd be better to be a negative value? const char Pattern[] = {0}; - auto URes = m_Dditable.Enqueue.pfnUSMFill( + auto URes = context.urDdiTable.Enqueue.pfnUSMFill( Queue, (void *)MappedPtr, 1, Pattern, PageSize, NumEventsInWaitList, EventsWaitList, Event); if (URes != UR_RESULT_SUCCESS) { @@ -378,7 +379,7 @@ ur_result_t SanitizerInterceptor::piextEnqueueMemSetShadow( } const char Pattern[] = {(char)Value}; - auto URes = m_Dditable.Enqueue.pfnUSMFill( + auto URes = context.urDdiTable.Enqueue.pfnUSMFill( Queue, (void *)ShadowBegin, 1, Pattern, (ShadowEnd - ShadowBegin + 1), NumEventsInWaitList, EventsWaitList, Event); @@ -393,22 +394,12 @@ ur_result_t SanitizerInterceptor::piextEnqueueMemSetShadow( return UR_RESULT_SUCCESS; } -ur_result_t SanitizerInterceptor::enqueuePoisonShadow( - ur_context_handle_t Context, ur_device_handle_t Device, - ur_queue_handle_t Queue, uptr Addr, uptr Size, u8 Value, - ur_event_handle_t DepEvent, ur_event_handle_t *OutEvent) { - uint32_t NumEvents = DepEvent ? 1 : 0; - const ur_event_handle_t *EventsList = DepEvent ? &DepEvent : nullptr; - return piextEnqueueMemSetShadow(Context, Device, Queue, (void *)Addr, Size, - Value, NumEvents, EventsList, OutEvent); -} - ur_result_t SanitizerInterceptor::enqueueAllocInfo( ur_context_handle_t Context, ur_device_handle_t Device, ur_queue_handle_t Queue, USMAllocInfo &AllocInfo, ur_event_handle_t &LastEvent) { // Init zero - UR_CALL(enqueuePoisonShadow(Context, Device, Queue, AllocInfo.AllocBegin, + UR_CALL(enqueueMemSetShadow(Context, Device, Queue, AllocInfo.AllocBegin, AllocInfo.AllocSize, 0, LastEvent, &LastEvent)); uptr TailBegin = RoundUpTo(AllocInfo.UserEnd, ASAN_SHADOW_GRANULARITY); @@ -418,7 +409,7 @@ ur_result_t SanitizerInterceptor::enqueueAllocInfo( if (TailBegin != AllocInfo.UserEnd) { auto Value = AllocInfo.UserEnd - RoundDownTo(AllocInfo.UserEnd, ASAN_SHADOW_GRANULARITY); - UR_CALL(enqueuePoisonShadow(Context, Device, Queue, AllocInfo.UserEnd, + UR_CALL(enqueueMemSetShadow(Context, Device, Queue, AllocInfo.UserEnd, 1, Value, LastEvent, &LastEvent)); } @@ -442,12 +433,12 @@ ur_result_t SanitizerInterceptor::enqueueAllocInfo( } // Left red zone - UR_CALL(enqueuePoisonShadow(Context, Device, Queue, AllocInfo.AllocBegin, + UR_CALL(enqueueMemSetShadow(Context, Device, Queue, AllocInfo.AllocBegin, AllocInfo.UserBegin - AllocInfo.AllocBegin, ShadowByte, LastEvent, &LastEvent)); // Right red zone - UR_CALL(enqueuePoisonShadow(Context, Device, Queue, TailBegin, + UR_CALL(enqueueMemSetShadow(Context, Device, Queue, TailBegin, TailEnd - TailBegin, ShadowByte, LastEvent, &LastEvent)); @@ -455,8 +446,8 @@ ur_result_t SanitizerInterceptor::enqueueAllocInfo( } ur_result_t SanitizerInterceptor::updateShadowMemory(ur_queue_handle_t Queue) { - auto Context = getContext(Queue, m_Dditable); - auto Device = getDevice(Queue, m_Dditable); + auto Context = getContext(Queue); + auto Device = getDevice(Queue); assert(Device != nullptr); auto &ContextInfo = getContextInfo(Context); @@ -517,7 +508,7 @@ ur_result_t SanitizerInterceptor::addDevice(ur_context_handle_t Context, // Query device type ur_device_type_t DeviceType; - UR_CALL(m_Dditable.Device.pfnGetInfo( + UR_CALL(context.urDdiTable.Device.pfnGetInfo( Device, UR_DEVICE_INFO_TYPE, sizeof(DeviceType), &DeviceType, nullptr)); switch (DeviceType) { case UR_DEVICE_TYPE_CPU: @@ -531,7 +522,7 @@ ur_result_t SanitizerInterceptor::addDevice(ur_context_handle_t Context, } // Query alignment - UR_CALL(m_Dditable.Device.pfnGetInfo( + UR_CALL(context.urDdiTable.Device.pfnGetInfo( Device, UR_DEVICE_INFO_MEM_BASE_ADDR_ALIGN, sizeof(DeviceInfoPtr->Alignment), &DeviceInfoPtr->Alignment, nullptr)); @@ -559,9 +550,9 @@ ur_result_t SanitizerInterceptor::addQueue(ur_context_handle_t Context, void SanitizerInterceptor::prepareLaunch(ur_queue_handle_t Queue, ur_kernel_handle_t Kernel) { - auto Context = getContext(Queue, m_Dditable); - auto Device = getDevice(Queue, m_Dditable); - auto Program = getProgram(Kernel, m_Dditable); + auto Context = getContext(Queue); + auto Device = getDevice(Queue); + auto Program = getProgram(Kernel); auto &ContextInfo = getContextInfo(Context); auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); @@ -577,9 +568,10 @@ void SanitizerInterceptor::prepareLaunch(ur_queue_handle_t Queue, uint32_t NumEvents = LastEvent ? 1 : 0; const ur_event_handle_t *EventsList = LastEvent ? &LastEvent : nullptr; - auto Result = m_Dditable.Enqueue.pfnDeviceGlobalVariableWrite( - Queue, Program, Name, false, sizeof(uptr), 0, Value, NumEvents, - EventsList, &NewEvent); + auto Result = + context.urDdiTable.Enqueue.pfnDeviceGlobalVariableWrite( + Queue, Program, Name, false, sizeof(uptr), 0, Value, + NumEvents, EventsList, &NewEvent); if (Result != UR_RESULT_SUCCESS) { context.logger.warning("Device Global Write Failed[{}]: {}", Name, getUrResultString(Result)); diff --git a/source/loader/layers/sanitizer/asan_interceptor.hpp b/source/loader/layers/sanitizer/asan_interceptor.hpp index b5f6508345..d60129dd64 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.hpp +++ b/source/loader/layers/sanitizer/asan_interceptor.hpp @@ -83,7 +83,7 @@ struct ContextInfo { class SanitizerInterceptor { public: - SanitizerInterceptor(ur_dditable_t &dditable) : m_Dditable(dditable) {} + SanitizerInterceptor() {} ur_result_t allocateMemory(ur_context_handle_t Context, ur_device_handle_t Device, @@ -116,14 +116,7 @@ class SanitizerInterceptor { std::string getKernelName(ur_kernel_handle_t Kernel); ur_result_t allocShadowMemory(ur_context_handle_t Context, DeviceInfo &DeviceInfo); - ur_result_t piextEnqueueMemSetShadow(ur_context_handle_t Context, - ur_device_handle_t Device, - ur_queue_handle_t Queue, void *Addr, - size_t Size, uint8_t Value, - size_t NumEvents, - const ur_event_handle_t *EventsList, - ur_event_handle_t *OutEvent); - ur_result_t enqueuePoisonShadow(ur_context_handle_t Context, + ur_result_t enqueueMemSetShadow(ur_context_handle_t Context, ur_device_handle_t Device, ur_queue_handle_t Queue, uptr Addr, uptr Size, u8 Value, @@ -140,8 +133,6 @@ class SanitizerInterceptor { std::unordered_map> m_ContextMap; ur_shared_mutex m_ContextMapMutex; - - ur_dditable_t &m_Dditable; }; } // namespace ur_san_layer diff --git a/source/loader/layers/sanitizer/ur_san_layer.cpp b/source/loader/layers/sanitizer/ur_san_layer.cpp index bfec1e3541..330856f9dd 100644 --- a/source/loader/layers/sanitizer/ur_san_layer.cpp +++ b/source/loader/layers/sanitizer/ur_san_layer.cpp @@ -19,7 +19,7 @@ context_t context; /////////////////////////////////////////////////////////////////////////////// context_t::context_t() - : interceptor(new SanitizerInterceptor(urDdiTable)), + : interceptor(new SanitizerInterceptor()), logger(logger::create_logger("sanitizer")) {} bool context_t::isAvailable() const { return true; } From 61c0ca9dbcee4d6d93af05ddaec754a0460879e8 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Tue, 14 Nov 2023 00:28:13 -0800 Subject: [PATCH 38/72] UR: llvm license --- .../layers/sanitizer/asan_interceptor.cpp | 18 +++++++--------- source/loader/layers/sanitizer/common.hpp | 21 ++++++++++--------- .../sanitizer/device_sanitizer_report.hpp | 11 ++++++++++ 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index 24e9c434ad..cfb3416feb 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -1,9 +1,3 @@ -//==---------- sanitizer_interceptor.cpp - Sanitizer interceptor -----------==// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// //===----------------------------------------------------------------------===// /* * @@ -16,7 +10,12 @@ * @file ur_san_layer.cpp * */ - +//==---------- sanitizer_interceptor.cpp - Sanitizer interceptor -----------==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// #include "asan_interceptor.hpp" #include "device_sanitizer_report.hpp" #include "ur_san_layer.hpp" @@ -107,7 +106,7 @@ ur_result_t SanitizerInterceptor::allocateMemory( Alignment = DeviceInfo.Alignment; } - // Calcuate Size + Red Zone Size + // Copy from LLVM compiler-rt/lib/asan uptr RZLog = ComputeRZLog(Size); uptr RZSize = RZLog2Size(RZLog); uptr RoundedSize = RoundUpTo(Size, Alignment); @@ -128,6 +127,7 @@ ur_result_t SanitizerInterceptor::allocateMemory( die("SanitizerInterceptor: unsupport memory type"); } + // Copy from LLVM compiler-rt/lib/asan uptr AllocBegin = reinterpret_cast(Allocated); uptr AllocEnd = AllocBegin + NeededSize; uptr UserBegin = AllocBegin + RZSize; @@ -165,8 +165,6 @@ ur_result_t SanitizerInterceptor::releaseMemory(ur_context_handle_t Context, void *Ptr) { auto &ContextInfo = getContextInfo(Context); - context.logger.debug("ReleaseMemory: {}", Ptr); - std::shared_lock Guard(ContextInfo.Mutex); auto Addr = (uptr)Ptr; diff --git a/source/loader/layers/sanitizer/common.hpp b/source/loader/layers/sanitizer/common.hpp index cedc29ccd1..dbacc81700 100644 --- a/source/loader/layers/sanitizer/common.hpp +++ b/source/loader/layers/sanitizer/common.hpp @@ -1,10 +1,3 @@ -//==---------- common.hpp - Device Sanitizer -----------==// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// /* * * Copyright (C) 2023 Intel Corporation @@ -16,6 +9,13 @@ * @file common.hpp * */ +//==---------- common.hpp - Device Sanitizer -----------==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// #pragma once #include @@ -26,6 +26,9 @@ namespace ur_san_layer { +// ================================================================ +// Copy from LLVM compiler-rt/lib/asan + typedef uintptr_t uptr; typedef unsigned char u8; typedef unsigned int u32; @@ -70,9 +73,7 @@ inline constexpr uptr ComputeRZLog(uptr user_requested_size) { return rz_log; } -inline constexpr uptr MemToShadow(uptr Addr, uptr ShadowOffset) { - return ShadowOffset + ((Addr) >> ASAN_SHADOW_SCALE); -} +// ================================================================ static auto getUrResultString = [](ur_result_t Result) { switch (Result) { diff --git a/source/loader/layers/sanitizer/device_sanitizer_report.hpp b/source/loader/layers/sanitizer/device_sanitizer_report.hpp index 04318d280f..3762fdd57a 100644 --- a/source/loader/layers/sanitizer/device_sanitizer_report.hpp +++ b/source/loader/layers/sanitizer/device_sanitizer_report.hpp @@ -1,3 +1,14 @@ +/* + * + * Copyright (C) 2023 Intel Corporation + * + * Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. + * See LICENSE.TXT + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + * @file common.hpp + * + */ //==---------- device_sanitizer_report.hpp - Device Sanitizer -----------==// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. From 0ad1bb177906a39e6f14106dd96773048afc7b8d Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Tue, 14 Nov 2023 00:36:35 -0800 Subject: [PATCH 39/72] UR: llvm license --- source/loader/layers/sanitizer/asan_interceptor.cpp | 10 ++-------- source/loader/layers/sanitizer/asan_interceptor.hpp | 1 + source/loader/layers/sanitizer/common.hpp | 8 +------- .../layers/sanitizer/device_sanitizer_report.hpp | 10 ++-------- source/loader/layers/sanitizer/ur_san_layer.cpp | 1 + 5 files changed, 7 insertions(+), 23 deletions(-) diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index cfb3416feb..42afeb502f 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -10,12 +10,7 @@ * @file ur_san_layer.cpp * */ -//==---------- sanitizer_interceptor.cpp - Sanitizer interceptor -----------==// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// + #include "asan_interceptor.hpp" #include "device_sanitizer_report.hpp" #include "ur_san_layer.hpp" @@ -197,7 +192,7 @@ bool SanitizerInterceptor::preLaunchKernel(ur_kernel_handle_t Kernel, ur_event_handle_t &Event) { prepareLaunch(Queue, Kernel); - updateShadowMemory(Queue); + UR_CALL(updateShadowMemory(Queue)); // Return LastEvent in QueueInfo auto Context = getContext(Queue); @@ -216,7 +211,6 @@ void SanitizerInterceptor::postLaunchKernel(ur_kernel_handle_t Kernel, ur_event_handle_t *Event, bool SetCallback) { auto Program = getProgram(Kernel); - ur_event_handle_t ReadEvent{}; // If kernel has defined SPIR_DeviceSanitizerReportMem, then we try to read it diff --git a/source/loader/layers/sanitizer/asan_interceptor.hpp b/source/loader/layers/sanitizer/asan_interceptor.hpp index d60129dd64..359bafbfdf 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.hpp +++ b/source/loader/layers/sanitizer/asan_interceptor.hpp @@ -9,6 +9,7 @@ * @file ur_san_layer.cpp * */ + #pragma once #include "common.hpp" diff --git a/source/loader/layers/sanitizer/common.hpp b/source/loader/layers/sanitizer/common.hpp index dbacc81700..58a50d5a1b 100644 --- a/source/loader/layers/sanitizer/common.hpp +++ b/source/loader/layers/sanitizer/common.hpp @@ -9,13 +9,7 @@ * @file common.hpp * */ -//==---------- common.hpp - Device Sanitizer -----------==// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// + #pragma once #include diff --git a/source/loader/layers/sanitizer/device_sanitizer_report.hpp b/source/loader/layers/sanitizer/device_sanitizer_report.hpp index 3762fdd57a..f23ea1d647 100644 --- a/source/loader/layers/sanitizer/device_sanitizer_report.hpp +++ b/source/loader/layers/sanitizer/device_sanitizer_report.hpp @@ -6,16 +6,10 @@ * See LICENSE.TXT * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception * - * @file common.hpp + * @file device_sanitizer_report.hpp * */ -//==---------- device_sanitizer_report.hpp - Device Sanitizer -----------==// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// + #pragma once #include diff --git a/source/loader/layers/sanitizer/ur_san_layer.cpp b/source/loader/layers/sanitizer/ur_san_layer.cpp index 330856f9dd..11b2725e0c 100644 --- a/source/loader/layers/sanitizer/ur_san_layer.cpp +++ b/source/loader/layers/sanitizer/ur_san_layer.cpp @@ -9,6 +9,7 @@ * @file ur_san_layer.cpp * */ + #include "ur_san_layer.hpp" #include "asan_interceptor.hpp" #include "ur_api.h" From cc4402a8aec6d227d9841258f4d608fb78dd78c3 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Tue, 14 Nov 2023 00:38:25 -0800 Subject: [PATCH 40/72] UR: once --- source/loader/layers/sanitizer/ur_san_layer.hpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/source/loader/layers/sanitizer/ur_san_layer.hpp b/source/loader/layers/sanitizer/ur_san_layer.hpp index e9ec44e442..8fc37e2dfb 100644 --- a/source/loader/layers/sanitizer/ur_san_layer.hpp +++ b/source/loader/layers/sanitizer/ur_san_layer.hpp @@ -10,8 +10,7 @@ * */ -#ifndef UR_SANITIZER_LAYER_H -#define UR_SANITIZER_LAYER_H 1 +#pragma once #include "logger/ur_logger.hpp" #include "ur_ddi.h" @@ -53,5 +52,3 @@ class __urdlllocal context_t : public proxy_layer_context_t { extern context_t context; } // namespace ur_san_layer - -#endif /* UR_sanitizer_LAYER_H */ From bf358c10e1ed1956c9b3a95c2e96ad3173ea5cff Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Tue, 14 Nov 2023 00:45:22 -0800 Subject: [PATCH 41/72] UR: clean code --- source/loader/layers/sanitizer/ur_sanddi.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/loader/layers/sanitizer/ur_sanddi.cpp b/source/loader/layers/sanitizer/ur_sanddi.cpp index 98c10155c4..b85a5633de 100644 --- a/source/loader/layers/sanitizer/ur_sanddi.cpp +++ b/source/loader/layers/sanitizer/ur_sanddi.cpp @@ -400,8 +400,9 @@ ur_result_t context_t::init(ur_dditable_t *dditable, return result; } - // FIXME: Just copy needed APIs? + // FIXME: Interceptor needs use some of APIs that aren't registered, maybe just copy needed APIs? urDdiTable = *dditable; + if (UR_RESULT_SUCCESS == result) { result = ur_san_layer::urGetContextProcAddrTable(UR_API_VERSION_CURRENT, &dditable->Context); From d5e7946466d7c3a52630d4c0c144ca32f27248ae Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Tue, 14 Nov 2023 00:57:17 -0800 Subject: [PATCH 42/72] UR: clean includes --- source/loader/layers/sanitizer/asan_interceptor.cpp | 4 ---- source/loader/layers/sanitizer/asan_interceptor.hpp | 1 - source/loader/layers/sanitizer/common.hpp | 6 +++--- source/loader/layers/sanitizer/device_sanitizer_report.hpp | 2 +- source/loader/layers/sanitizer/ur_san_layer.cpp | 2 -- source/loader/layers/sanitizer/ur_san_layer.hpp | 2 -- 6 files changed, 4 insertions(+), 13 deletions(-) diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index 42afeb502f..6f6c593aec 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -15,10 +15,6 @@ #include "device_sanitizer_report.hpp" #include "ur_san_layer.hpp" -#include -#include -#include - namespace ur_san_layer { namespace { diff --git a/source/loader/layers/sanitizer/asan_interceptor.hpp b/source/loader/layers/sanitizer/asan_interceptor.hpp index 359bafbfdf..a57ead5d28 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.hpp +++ b/source/loader/layers/sanitizer/asan_interceptor.hpp @@ -14,7 +14,6 @@ #include "common.hpp" -#include #include #include #include diff --git a/source/loader/layers/sanitizer/common.hpp b/source/loader/layers/sanitizer/common.hpp index 58a50d5a1b..6bd523723b 100644 --- a/source/loader/layers/sanitizer/common.hpp +++ b/source/loader/layers/sanitizer/common.hpp @@ -12,12 +12,12 @@ #pragma once +#include "ur/ur.hpp" +#include "ur_ddi.h" + #include #include -#include "ur_ddi.h" -#include - namespace ur_san_layer { // ================================================================ diff --git a/source/loader/layers/sanitizer/device_sanitizer_report.hpp b/source/loader/layers/sanitizer/device_sanitizer_report.hpp index f23ea1d647..640e3be4ab 100644 --- a/source/loader/layers/sanitizer/device_sanitizer_report.hpp +++ b/source/loader/layers/sanitizer/device_sanitizer_report.hpp @@ -40,7 +40,7 @@ struct DeviceSanitizerReport { int Flag = 0; char File[256 + 1] = ""; - char Func[128 + 1] = ""; + char Func[256 + 1] = ""; int32_t Line = 0; diff --git a/source/loader/layers/sanitizer/ur_san_layer.cpp b/source/loader/layers/sanitizer/ur_san_layer.cpp index 11b2725e0c..00b4e2f20d 100644 --- a/source/loader/layers/sanitizer/ur_san_layer.cpp +++ b/source/loader/layers/sanitizer/ur_san_layer.cpp @@ -12,8 +12,6 @@ #include "ur_san_layer.hpp" #include "asan_interceptor.hpp" -#include "ur_api.h" -#include "ur_util.hpp" namespace ur_san_layer { context_t context; diff --git a/source/loader/layers/sanitizer/ur_san_layer.hpp b/source/loader/layers/sanitizer/ur_san_layer.hpp index 8fc37e2dfb..78145ba205 100644 --- a/source/loader/layers/sanitizer/ur_san_layer.hpp +++ b/source/loader/layers/sanitizer/ur_san_layer.hpp @@ -13,9 +13,7 @@ #pragma once #include "logger/ur_logger.hpp" -#include "ur_ddi.h" #include "ur_proxy_layer.hpp" -#include "ur_util.hpp" #define SANITIZER_COMP_NAME "sanitizer layer" From 50666b7064d3cafd6c93327b83a5e5a55e05be4d Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Tue, 14 Nov 2023 01:08:50 -0800 Subject: [PATCH 43/72] UR: clean comment --- source/loader/layers/sanitizer/device_sanitizer_report.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/loader/layers/sanitizer/device_sanitizer_report.hpp b/source/loader/layers/sanitizer/device_sanitizer_report.hpp index 640e3be4ab..411d308820 100644 --- a/source/loader/layers/sanitizer/device_sanitizer_report.hpp +++ b/source/loader/layers/sanitizer/device_sanitizer_report.hpp @@ -34,8 +34,6 @@ enum class DeviceSanitizerMemoryType : int32_t { MEM_BUFFER, }; -// NOTE Layout of this structure should be aligned with the one in -// sycl/include/sycl/detail/device_sanitizer_report.hpp struct DeviceSanitizerReport { int Flag = 0; From de030965c5f83b866bc70dea729c0255fe8d4f7d Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Tue, 14 Nov 2023 01:16:00 -0800 Subject: [PATCH 44/72] UR: rename files --- source/loader/CMakeLists.txt | 4 +- .../layers/sanitizer/asan_interceptor.cpp | 8 ++-- .../layers/sanitizer/asan_interceptor.hpp | 6 +-- source/loader/layers/sanitizer/common.hpp | 4 +- .../sanitizer/device_sanitizer_report.hpp | 4 +- source/loader/layers/sanitizer/ur_sanddi.cpp | 48 +++++++++---------- ...r_san_layer.cpp => ur_sanitizer_layer.cpp} | 8 ++-- ...r_san_layer.hpp => ur_sanitizer_layer.hpp} | 6 +-- source/loader/ur_lib.hpp | 4 +- 9 files changed, 46 insertions(+), 46 deletions(-) rename source/loader/layers/sanitizer/{ur_san_layer.cpp => ur_sanitizer_layer.cpp} (83%) rename source/loader/layers/sanitizer/{ur_san_layer.hpp => ur_sanitizer_layer.hpp} (92%) diff --git a/source/loader/CMakeLists.txt b/source/loader/CMakeLists.txt index 43978c5aa6..be0587b708 100644 --- a/source/loader/CMakeLists.txt +++ b/source/loader/CMakeLists.txt @@ -108,8 +108,8 @@ if(UR_ENABLE_SANITIZER) ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/asan_interceptor.hpp ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/common.hpp ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/device_sanitizer_report.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/ur_san_layer.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/ur_san_layer.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/ur_sanitizer_layer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/ur_sanitizer_layer.hpp ${CMAKE_CURRENT_SOURCE_DIR}/layers/sanitizer/ur_sanddi.cpp ) target_include_directories(ur_loader PRIVATE diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index 6f6c593aec..45e1e61814 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -7,15 +7,15 @@ * See LICENSE.TXT * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception * - * @file ur_san_layer.cpp + * @file ur_sanitizer_layer.cpp * */ #include "asan_interceptor.hpp" #include "device_sanitizer_report.hpp" -#include "ur_san_layer.hpp" +#include "ur_sanitizer_layer.hpp" -namespace ur_san_layer { +namespace ur_sanitizer_layer { namespace { @@ -579,4 +579,4 @@ void SanitizerInterceptor::prepareLaunch(ur_queue_handle_t Queue, QueueInfo.LastEvent = LastEvent; } -} // namespace ur_san_layer +} // namespace ur_sanitizer_layer diff --git a/source/loader/layers/sanitizer/asan_interceptor.hpp b/source/loader/layers/sanitizer/asan_interceptor.hpp index a57ead5d28..e83378deae 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.hpp +++ b/source/loader/layers/sanitizer/asan_interceptor.hpp @@ -6,7 +6,7 @@ * See LICENSE.TXT * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception * - * @file ur_san_layer.cpp + * @file ur_sanitizer_layer.cpp * */ @@ -19,7 +19,7 @@ #include #include -namespace ur_san_layer { +namespace ur_sanitizer_layer { enum USMMemoryType { DEVICE, SHARE, HOST, MEM_BUFFER }; @@ -135,4 +135,4 @@ class SanitizerInterceptor { ur_shared_mutex m_ContextMapMutex; }; -} // namespace ur_san_layer +} // namespace ur_sanitizer_layer diff --git a/source/loader/layers/sanitizer/common.hpp b/source/loader/layers/sanitizer/common.hpp index 6bd523723b..24ad9dabf6 100644 --- a/source/loader/layers/sanitizer/common.hpp +++ b/source/loader/layers/sanitizer/common.hpp @@ -18,7 +18,7 @@ #include #include -namespace ur_san_layer { +namespace ur_sanitizer_layer { // ================================================================ // Copy from LLVM compiler-rt/lib/asan @@ -225,4 +225,4 @@ static auto getUrResultString = [](ur_result_t Result) { return Result; \ } -} // namespace ur_san_layer +} // namespace ur_sanitizer_layer diff --git a/source/loader/layers/sanitizer/device_sanitizer_report.hpp b/source/loader/layers/sanitizer/device_sanitizer_report.hpp index 411d308820..abe2949db4 100644 --- a/source/loader/layers/sanitizer/device_sanitizer_report.hpp +++ b/source/loader/layers/sanitizer/device_sanitizer_report.hpp @@ -14,7 +14,7 @@ #include -namespace ur_san_layer { +namespace ur_sanitizer_layer { enum class DeviceSanitizerErrorType : int32_t { UNKNOWN, @@ -92,4 +92,4 @@ const char *DeviceSanitizerFormat(DeviceSanitizerErrorType ErrorType) { } } -} // namespace ur_san_layer +} // namespace ur_sanitizer_layer diff --git a/source/loader/layers/sanitizer/ur_sanddi.cpp b/source/loader/layers/sanitizer/ur_sanddi.cpp index b85a5633de..7245f5339f 100644 --- a/source/loader/layers/sanitizer/ur_sanddi.cpp +++ b/source/loader/layers/sanitizer/ur_sanddi.cpp @@ -11,9 +11,9 @@ */ #include "asan_interceptor.hpp" -#include "ur_san_layer.hpp" +#include "ur_sanitizer_layer.hpp" -namespace ur_san_layer { +namespace ur_sanitizer_layer { /////////////////////////////////////////////////////////////////////////////// /// @brief Intercept function for urUSMHostAlloc @@ -276,19 +276,19 @@ __urdlllocal ur_result_t UR_APICALL urGetContextProcAddrTable( return UR_RESULT_ERROR_INVALID_NULL_POINTER; } - if (UR_MAJOR_VERSION(ur_san_layer::context.version) != + if (UR_MAJOR_VERSION(ur_sanitizer_layer::context.version) != UR_MAJOR_VERSION(version) || - UR_MINOR_VERSION(ur_san_layer::context.version) > + UR_MINOR_VERSION(ur_sanitizer_layer::context.version) > UR_MINOR_VERSION(version)) { return UR_RESULT_ERROR_UNSUPPORTED_VERSION; } ur_result_t result = UR_RESULT_SUCCESS; - pDdiTable->pfnCreate = ur_san_layer::urContextCreate; + pDdiTable->pfnCreate = ur_sanitizer_layer::urContextCreate; pDdiTable->pfnCreateWithNativeHandle = - ur_san_layer::urContextCreateWithNativeHandle; + ur_sanitizer_layer::urContextCreateWithNativeHandle; return result; } @@ -309,16 +309,16 @@ __urdlllocal ur_result_t UR_APICALL urGetEnqueueProcAddrTable( return UR_RESULT_ERROR_INVALID_NULL_POINTER; } - if (UR_MAJOR_VERSION(ur_san_layer::context.version) != + if (UR_MAJOR_VERSION(ur_sanitizer_layer::context.version) != UR_MAJOR_VERSION(version) || - UR_MINOR_VERSION(ur_san_layer::context.version) > + UR_MINOR_VERSION(ur_sanitizer_layer::context.version) > UR_MINOR_VERSION(version)) { return UR_RESULT_ERROR_UNSUPPORTED_VERSION; } ur_result_t result = UR_RESULT_SUCCESS; - pDdiTable->pfnKernelLaunch = ur_san_layer::urEnqueueKernelLaunch; + pDdiTable->pfnKernelLaunch = ur_sanitizer_layer::urEnqueueKernelLaunch; return result; } @@ -339,16 +339,16 @@ __urdlllocal ur_result_t UR_APICALL urGetQueueProcAddrTable( return UR_RESULT_ERROR_INVALID_NULL_POINTER; } - if (UR_MAJOR_VERSION(ur_san_layer::context.version) != + if (UR_MAJOR_VERSION(ur_sanitizer_layer::context.version) != UR_MAJOR_VERSION(version) || - UR_MINOR_VERSION(ur_san_layer::context.version) > + UR_MINOR_VERSION(ur_sanitizer_layer::context.version) > UR_MINOR_VERSION(version)) { return UR_RESULT_ERROR_UNSUPPORTED_VERSION; } ur_result_t result = UR_RESULT_SUCCESS; - pDdiTable->pfnCreate = ur_san_layer::urQueueCreate; + pDdiTable->pfnCreate = ur_sanitizer_layer::urQueueCreate; return result; } @@ -369,16 +369,16 @@ __urdlllocal ur_result_t UR_APICALL urGetUSMProcAddrTable( return UR_RESULT_ERROR_INVALID_NULL_POINTER; } - if (UR_MAJOR_VERSION(ur_san_layer::context.version) != + if (UR_MAJOR_VERSION(ur_sanitizer_layer::context.version) != UR_MAJOR_VERSION(version) || - UR_MINOR_VERSION(ur_san_layer::context.version) > + UR_MINOR_VERSION(ur_sanitizer_layer::context.version) > UR_MINOR_VERSION(version)) { return UR_RESULT_ERROR_UNSUPPORTED_VERSION; } ur_result_t result = UR_RESULT_SUCCESS; - pDdiTable->pfnDeviceAlloc = ur_san_layer::urUSMDeviceAlloc; + pDdiTable->pfnDeviceAlloc = ur_sanitizer_layer::urUSMDeviceAlloc; return result; } @@ -404,25 +404,25 @@ ur_result_t context_t::init(ur_dditable_t *dditable, urDdiTable = *dditable; if (UR_RESULT_SUCCESS == result) { - result = ur_san_layer::urGetContextProcAddrTable(UR_API_VERSION_CURRENT, - &dditable->Context); + result = ur_sanitizer_layer::urGetContextProcAddrTable( + UR_API_VERSION_CURRENT, &dditable->Context); } if (UR_RESULT_SUCCESS == result) { - result = ur_san_layer::urGetEnqueueProcAddrTable(UR_API_VERSION_CURRENT, - &dditable->Enqueue); + result = ur_sanitizer_layer::urGetEnqueueProcAddrTable( + UR_API_VERSION_CURRENT, &dditable->Enqueue); } if (UR_RESULT_SUCCESS == result) { - result = ur_san_layer::urGetQueueProcAddrTable(UR_API_VERSION_CURRENT, - &dditable->Queue); + result = ur_sanitizer_layer::urGetQueueProcAddrTable( + UR_API_VERSION_CURRENT, &dditable->Queue); } if (UR_RESULT_SUCCESS == result) { - result = ur_san_layer::urGetUSMProcAddrTable(UR_API_VERSION_CURRENT, - &dditable->USM); + result = ur_sanitizer_layer::urGetUSMProcAddrTable( + UR_API_VERSION_CURRENT, &dditable->USM); } return result; } -} // namespace ur_san_layer +} // namespace ur_sanitizer_layer diff --git a/source/loader/layers/sanitizer/ur_san_layer.cpp b/source/loader/layers/sanitizer/ur_sanitizer_layer.cpp similarity index 83% rename from source/loader/layers/sanitizer/ur_san_layer.cpp rename to source/loader/layers/sanitizer/ur_sanitizer_layer.cpp index 00b4e2f20d..84a064997d 100644 --- a/source/loader/layers/sanitizer/ur_san_layer.cpp +++ b/source/loader/layers/sanitizer/ur_sanitizer_layer.cpp @@ -6,14 +6,14 @@ * See LICENSE.TXT * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception * - * @file ur_san_layer.cpp + * @file ur_sanitizer_layer.cpp * */ -#include "ur_san_layer.hpp" +#include "ur_sanitizer_layer.hpp" #include "asan_interceptor.hpp" -namespace ur_san_layer { +namespace ur_sanitizer_layer { context_t context; /////////////////////////////////////////////////////////////////////////////// @@ -25,4 +25,4 @@ bool context_t::isAvailable() const { return true; } /////////////////////////////////////////////////////////////////////////////// context_t::~context_t() {} -} // namespace ur_san_layer +} // namespace ur_sanitizer_layer diff --git a/source/loader/layers/sanitizer/ur_san_layer.hpp b/source/loader/layers/sanitizer/ur_sanitizer_layer.hpp similarity index 92% rename from source/loader/layers/sanitizer/ur_san_layer.hpp rename to source/loader/layers/sanitizer/ur_sanitizer_layer.hpp index 78145ba205..7e191b3785 100644 --- a/source/loader/layers/sanitizer/ur_san_layer.hpp +++ b/source/loader/layers/sanitizer/ur_sanitizer_layer.hpp @@ -6,7 +6,7 @@ * See LICENSE.TXT * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception * - * @file ur_san_layer.hpp + * @file ur_sanitizer_layer.hpp * */ @@ -17,7 +17,7 @@ #define SANITIZER_COMP_NAME "sanitizer layer" -namespace ur_san_layer { +namespace ur_sanitizer_layer { class SanitizerInterceptor; @@ -49,4 +49,4 @@ class __urdlllocal context_t : public proxy_layer_context_t { }; extern context_t context; -} // namespace ur_san_layer +} // namespace ur_sanitizer_layer diff --git a/source/loader/ur_lib.hpp b/source/loader/ur_lib.hpp index d95db8771d..12f3877932 100644 --- a/source/loader/ur_lib.hpp +++ b/source/loader/ur_lib.hpp @@ -23,7 +23,7 @@ #include "tracing/ur_tracing_layer.hpp" #endif #if UR_ENABLE_SANITIZER -#include "sanitizer/ur_san_layer.hpp" +#include "sanitizer/ur_sanitizer_layer.hpp" #endif #include @@ -72,7 +72,7 @@ class __urdlllocal context_t { &ur_tracing_layer::context, #endif #if UR_ENABLE_SANITIZER - &ur_san_layer::context + &ur_sanitizer_layer::context #endif }; std::string availableLayers; From b47ca29817abc20ce5e3490ae65d28ce244a55dd Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Tue, 14 Nov 2023 01:20:01 -0800 Subject: [PATCH 45/72] UR: update readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c156d4f079..27126a1340 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,7 @@ List of options provided by CMake: | UR_USE_UBSAN | Enable UndefinedBehavior Sanitizer | ON/OFF | OFF | | UR_USE_MSAN | Enable MemorySanitizer (clang only) | ON/OFF | OFF | | UR_ENABLE_TRACING | Enable XPTI-based tracing layer | ON/OFF | OFF | +| UR_ENABLE_SANITIZER | Enable device sanitizer layer | ON/OFF | OFF | | UR_CONFORMANCE_TARGET_TRIPLES | SYCL triples to build CTS device binaries for | Comma-separated list | spir64 | | UR_BUILD_ADAPTER_L0 | Fetch and use level-zero adapter from SYCL | ON/OFF | OFF | | UR_BUILD_ADAPTER_OPENCL | Fetch and use opencl adapter from SYCL | ON/OFF | OFF | From 1e2b6a0e80d346fcb2c7bd9115061832921b98ea Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Tue, 14 Nov 2023 22:47:57 -0800 Subject: [PATCH 46/72] UR: fix according to PR comments --- scripts/core/INTRO.rst | 15 +++++++++++++++ .../loader/layers/sanitizer/asan_interceptor.cpp | 14 +++++++------- source/loader/layers/sanitizer/common.hpp | 6 +++--- source/loader/layers/sanitizer/ur_sanddi.cpp | 11 +++++++++++ 4 files changed, 36 insertions(+), 10 deletions(-) diff --git a/scripts/core/INTRO.rst b/scripts/core/INTRO.rst index 4c3a1a9d2d..81aac51cb2 100644 --- a/scripts/core/INTRO.rst +++ b/scripts/core/INTRO.rst @@ -171,6 +171,15 @@ Unified Runtime loader implements tracing support through the `XPTI framework %s\n", #Call); \ + context.logger.debug("UR ---> {}", #Call); \ ur_result_t Result = (Call); \ if (PrintTrace) \ - fprintf(stderr, "UR <--- %s(%s)\n", #Call, \ - getUrResultString(Result)); \ + context.logger.debug("UR <--- {}({})", #Call, \ + getUrResultString(Result)); \ if (Result != UR_RESULT_SUCCESS) \ return Result; \ } diff --git a/source/loader/layers/sanitizer/ur_sanddi.cpp b/source/loader/layers/sanitizer/ur_sanddi.cpp index 7245f5339f..40af911cc9 100644 --- a/source/loader/layers/sanitizer/ur_sanddi.cpp +++ b/source/loader/layers/sanitizer/ur_sanddi.cpp @@ -400,6 +400,17 @@ ur_result_t context_t::init(ur_dditable_t *dditable, return result; } + if (context.enabledType == SanitizerType::AddressSanitizer) { + if (!(dditable->VirtualMem.pfnReserve && dditable->VirtualMem.pfnMap && + dditable->VirtualMem.pfnGranularityGetInfo)) { + die("Some VirtualMem APIs are needed to enable UR_LAYER_ASAN"); + } + + if (!dditable->PhysicalMem.pfnCreate) { + die("Some PhysicalMem APIs are needed to enable UR_LAYER_ASAN"); + } + } + // FIXME: Interceptor needs use some of APIs that aren't registered, maybe just copy needed APIs? urDdiTable = *dditable; From 42f67559e4f8e81fd5b0fab101e51188c2fa29dd Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Tue, 14 Nov 2023 23:29:05 -0800 Subject: [PATCH 47/72] UR: UR_ENABLE_SANITIZER=ON default --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f1e9e736d5..8ef4ef9a08 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,7 +33,7 @@ option(UR_USE_UBSAN "enable UndefinedBehaviorSanitizer" OFF) option(UR_USE_MSAN "enable MemorySanitizer" OFF) option(UR_USE_TSAN "enable ThreadSanitizer" OFF) option(UR_ENABLE_TRACING "enable api tracing through xpti" OFF) -option(UR_ENABLE_SANITIZER "enable device sanitizer" OFF) +option(UR_ENABLE_SANITIZER "enable device sanitizer" ON) option(UMF_BUILD_SHARED_LIBRARY "Build UMF as shared library" OFF) option(UMF_ENABLE_POOL_TRACKING "Build UMF with pool tracking" OFF) option(UR_BUILD_ADAPTER_L0 "build level 0 adapter from SYCL" OFF) From 5aaba09d21dec2675ea234c4d0f93d047792c7d6 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Mon, 20 Nov 2023 19:05:25 -0800 Subject: [PATCH 48/72] ignore VS clangd files --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 85770fe15c..89736ad22a 100644 --- a/.gitignore +++ b/.gitignore @@ -86,3 +86,7 @@ out/ # External content */**/external + +# VS clangd +/.cache +/compile_commands.json From 8dc90fdea420912a115e39e52eddfdd4795a71d6 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Mon, 20 Nov 2023 21:17:00 -0800 Subject: [PATCH 49/72] fix comments --- .../layers/sanitizer/asan_interceptor.cpp | 22 ++- source/loader/layers/sanitizer/common.hpp | 147 +----------------- .../layers/sanitizer/ur_sanitizer_layer.cpp | 2 +- .../layers/sanitizer/ur_sanitizer_layer.hpp | 2 +- 4 files changed, 13 insertions(+), 160 deletions(-) diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index 879c3b1c3e..45745be403 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -146,9 +146,9 @@ ur_result_t SanitizerInterceptor::allocateMemory( ContextInfo.AllocatedUSMMap[AllocBegin] = MemoryInfo; } - context.logger.info("AllocInfos:\n AllocBegin: {}\n User: {}-{}\n " - "NeededSize: {}\nType: {}", - AllocBegin, UserBegin, UserEnd, NeededSize, Type); + context.logger.info( + "AllocInfos(AllocBegin={}, User={}-{}, NeededSize={}, Type={})", + AllocBegin, UserBegin, UserEnd, NeededSize, Type); return UR_RESULT_SUCCESS; } @@ -325,8 +325,7 @@ ur_result_t SanitizerInterceptor::enqueueMemSetShadow( auto URes = context.urDdiTable.PhysicalMem.pfnCreate( Context, Device, PageSize, &Desc, &PhysicalMem); if (URes != UR_RESULT_SUCCESS) { - context.logger.error("zePhysicalMemCreate(): {}", - getUrResultString(URes)); + context.logger.error("zePhysicalMemCreate(): {}", URes); return URes; } } @@ -340,8 +339,7 @@ ur_result_t SanitizerInterceptor::enqueueMemSetShadow( Context, (void *)MappedPtr, PageSize, PhysicalMem, 0, UR_VIRTUAL_MEM_ACCESS_FLAG_READ_WRITE); if (URes != UR_RESULT_SUCCESS) { - context.logger.debug("zeVirtualMemMap(): %s", - getUrResultString(URes)); + context.logger.debug("zeVirtualMemMap(): {}", URes); } // Initialize to zero @@ -355,8 +353,7 @@ ur_result_t SanitizerInterceptor::enqueueMemSetShadow( Queue, (void *)MappedPtr, 1, Pattern, PageSize, NumEventsInWaitList, EventsWaitList, Event); if (URes != UR_RESULT_SUCCESS) { - context.logger.error("urEnqueueUSMFill(): {}", - getUrResultString(URes)); + context.logger.error("urEnqueueUSMFill(): {}", URes); return URes; } @@ -372,8 +369,7 @@ ur_result_t SanitizerInterceptor::enqueueMemSetShadow( (ShadowEnd - ShadowBegin + 1), NumEventsInWaitList, EventsWaitList, Event); if (URes != UR_RESULT_SUCCESS) { - context.logger.error("urEnqueueUSMFill(): {}", - getUrResultString(URes)); + context.logger.error("urEnqueueUSMFill(): {}", URes); return URes; } } else { @@ -561,8 +557,8 @@ void SanitizerInterceptor::prepareLaunch(ur_queue_handle_t Queue, Queue, Program, Name, false, sizeof(uptr), 0, Value, NumEvents, EventsList, &NewEvent); if (Result != UR_RESULT_SUCCESS) { - context.logger.warning("Device Global Write Failed[{}]: {}", - Name, getUrResultString(Result)); + context.logger.warning("Device Global[{}] Write Failed: {}", + Name, Result); return false; } LastEvent = NewEvent; diff --git a/source/loader/layers/sanitizer/common.hpp b/source/loader/layers/sanitizer/common.hpp index e7f3638732..0a783dbd5d 100644 --- a/source/loader/layers/sanitizer/common.hpp +++ b/source/loader/layers/sanitizer/common.hpp @@ -14,6 +14,7 @@ #include "ur/ur.hpp" #include "ur_ddi.h" +#include "ur_params.hpp" #include #include @@ -69,149 +70,6 @@ inline constexpr uptr ComputeRZLog(uptr user_requested_size) { // ================================================================ -static auto getUrResultString = [](ur_result_t Result) { - switch (Result) { - case UR_RESULT_SUCCESS: - return "UR_RESULT_SUCCESS"; - case UR_RESULT_ERROR_INVALID_OPERATION: - return "UR_RESULT_ERROR_INVALID_OPERATION"; - case UR_RESULT_ERROR_INVALID_QUEUE_PROPERTIES: - return "UR_RESULT_ERROR_INVALID_QUEUE_PROPERTIES"; - case UR_RESULT_ERROR_INVALID_QUEUE: - return "UR_RESULT_ERROR_INVALID_QUEUE"; - case UR_RESULT_ERROR_INVALID_VALUE: - return "UR_RESULT_ERROR_INVALID_VALUE"; - case UR_RESULT_ERROR_INVALID_CONTEXT: - return "UR_RESULT_ERROR_INVALID_CONTEXT"; - case UR_RESULT_ERROR_INVALID_PLATFORM: - return "UR_RESULT_ERROR_INVALID_PLATFORM"; - case UR_RESULT_ERROR_INVALID_BINARY: - return "UR_RESULT_ERROR_INVALID_BINARY"; - case UR_RESULT_ERROR_INVALID_PROGRAM: - return "UR_RESULT_ERROR_INVALID_PROGRAM"; - case UR_RESULT_ERROR_INVALID_SAMPLER: - return "UR_RESULT_ERROR_INVALID_SAMPLER"; - case UR_RESULT_ERROR_INVALID_BUFFER_SIZE: - return "UR_RESULT_ERROR_INVALID_BUFFER_SIZE"; - case UR_RESULT_ERROR_INVALID_MEM_OBJECT: - return "UR_RESULT_ERROR_INVALID_MEM_OBJECT"; - case UR_RESULT_ERROR_INVALID_EVENT: - return "UR_RESULT_ERROR_INVALID_EVENT"; - case UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST: - return "UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST"; - case UR_RESULT_ERROR_MISALIGNED_SUB_BUFFER_OFFSET: - return "UR_RESULT_ERROR_MISALIGNED_SUB_BUFFER_OFFSET"; - case UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE: - return "UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE"; - case UR_RESULT_ERROR_COMPILER_NOT_AVAILABLE: - return "UR_RESULT_ERROR_COMPILER_NOT_AVAILABLE"; - case UR_RESULT_ERROR_PROFILING_INFO_NOT_AVAILABLE: - return "UR_RESULT_ERROR_PROFILING_INFO_NOT_AVAILABLE"; - case UR_RESULT_ERROR_DEVICE_NOT_FOUND: - return "UR_RESULT_ERROR_DEVICE_NOT_FOUND"; - case UR_RESULT_ERROR_INVALID_DEVICE: - return "UR_RESULT_ERROR_INVALID_DEVICE"; - case UR_RESULT_ERROR_DEVICE_LOST: - return "UR_RESULT_ERROR_DEVICE_LOST"; - case UR_RESULT_ERROR_DEVICE_REQUIRES_RESET: - return "UR_RESULT_ERROR_DEVICE_REQUIRES_RESET"; - case UR_RESULT_ERROR_DEVICE_IN_LOW_POWER_STATE: - return "UR_RESULT_ERROR_DEVICE_IN_LOW_POWER_STATE"; - case UR_RESULT_ERROR_DEVICE_PARTITION_FAILED: - return "UR_RESULT_ERROR_DEVICE_PARTITION_FAILED"; - case UR_RESULT_ERROR_INVALID_DEVICE_PARTITION_COUNT: - return "UR_RESULT_ERROR_INVALID_DEVICE_PARTITION_COUNT"; - case UR_RESULT_ERROR_INVALID_WORK_ITEM_SIZE: - return "UR_RESULT_ERROR_INVALID_WORK_ITEM_SIZE"; - case UR_RESULT_ERROR_INVALID_WORK_DIMENSION: - return "UR_RESULT_ERROR_INVALID_WORK_DIMENSION"; - case UR_RESULT_ERROR_INVALID_KERNEL_ARGS: - return "UR_RESULT_ERROR_INVALID_KERNEL_ARGS"; - case UR_RESULT_ERROR_INVALID_KERNEL: - return "UR_RESULT_ERROR_INVALID_KERNEL"; - case UR_RESULT_ERROR_INVALID_KERNEL_NAME: - return "UR_RESULT_ERROR_INVALID_KERNEL_NAME"; - case UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX: - return "UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX"; - case UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_SIZE: - return "UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_SIZE"; - case UR_RESULT_ERROR_INVALID_KERNEL_ATTRIBUTE_VALUE: - return "UR_RESULT_ERROR_INVALID_KERNEL_ATTRIBUTE_VALUE"; - case UR_RESULT_ERROR_INVALID_IMAGE_SIZE: - return "UR_RESULT_ERROR_INVALID_IMAGE_SIZE"; - case UR_RESULT_ERROR_INVALID_IMAGE_FORMAT_DESCRIPTOR: - return "UR_RESULT_ERROR_INVALID_IMAGE_FORMAT_DESCRIPTOR"; - case UR_RESULT_ERROR_IMAGE_FORMAT_NOT_SUPPORTED: - return "UR_RESULT_ERROR_IMAGE_FORMAT_NOT_SUPPORTED"; - case UR_RESULT_ERROR_MEM_OBJECT_ALLOCATION_FAILURE: - return "UR_RESULT_ERROR_MEM_OBJECT_ALLOCATION_FAILURE"; - case UR_RESULT_ERROR_INVALID_PROGRAM_EXECUTABLE: - return "UR_RESULT_ERROR_INVALID_PROGRAM_EXECUTABLE"; - case UR_RESULT_ERROR_UNINITIALIZED: - return "UR_RESULT_ERROR_UNINITIALIZED"; - case UR_RESULT_ERROR_OUT_OF_HOST_MEMORY: - return "UR_RESULT_ERROR_OUT_OF_HOST_MEMORY"; - case UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY: - return "UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY"; - case UR_RESULT_ERROR_OUT_OF_RESOURCES: - return "UR_RESULT_ERROR_OUT_OF_RESOURCES"; - case UR_RESULT_ERROR_PROGRAM_BUILD_FAILURE: - return "UR_RESULT_ERROR_PROGRAM_BUILD_FAILURE"; - case UR_RESULT_ERROR_PROGRAM_LINK_FAILURE: - return "UR_RESULT_ERROR_PROGRAM_LINK_FAILURE"; - case UR_RESULT_ERROR_UNSUPPORTED_VERSION: - return "UR_RESULT_ERROR_UNSUPPORTED_VERSION"; - case UR_RESULT_ERROR_UNSUPPORTED_FEATURE: - return "UR_RESULT_ERROR_UNSUPPORTED_FEATURE"; - case UR_RESULT_ERROR_INVALID_ARGUMENT: - return "UR_RESULT_ERROR_INVALID_ARGUMENT"; - case UR_RESULT_ERROR_INVALID_NULL_HANDLE: - return "UR_RESULT_ERROR_INVALID_NULL_HANDLE"; - case UR_RESULT_ERROR_HANDLE_OBJECT_IN_USE: - return "UR_RESULT_ERROR_HANDLE_OBJECT_IN_USE"; - case UR_RESULT_ERROR_INVALID_NULL_POINTER: - return "UR_RESULT_ERROR_INVALID_NULL_POINTER"; - case UR_RESULT_ERROR_INVALID_SIZE: - return "UR_RESULT_ERROR_INVALID_SIZE"; - case UR_RESULT_ERROR_UNSUPPORTED_SIZE: - return "UR_RESULT_ERROR_UNSUPPORTED_SIZE"; - case UR_RESULT_ERROR_UNSUPPORTED_ALIGNMENT: - return "UR_RESULT_ERROR_UNSUPPORTED_ALIGNMENT"; - case UR_RESULT_ERROR_INVALID_SYNCHRONIZATION_OBJECT: - return "UR_RESULT_ERROR_INVALID_SYNCHRONIZATION_OBJECT"; - case UR_RESULT_ERROR_INVALID_ENUMERATION: - return "UR_RESULT_ERROR_INVALID_ENUMERATION"; - case UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION: - return "UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION"; - case UR_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT: - return "UR_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT"; - case UR_RESULT_ERROR_INVALID_NATIVE_BINARY: - return "UR_RESULT_ERROR_INVALID_NATIVE_BINARY"; - case UR_RESULT_ERROR_INVALID_GLOBAL_NAME: - return "UR_RESULT_ERROR_INVALID_GLOBAL_NAME"; - case UR_RESULT_ERROR_INVALID_FUNCTION_NAME: - return "UR_RESULT_ERROR_INVALID_FUNCTION_NAME"; - case UR_RESULT_ERROR_INVALID_GROUP_SIZE_DIMENSION: - return "UR_RESULT_ERROR_INVALID_GROUP_SIZE_DIMENSION"; - case UR_RESULT_ERROR_INVALID_GLOBAL_WIDTH_DIMENSION: - return "UR_RESULT_ERROR_INVALID_GLOBAL_WIDTH_DIMENSION"; - case UR_RESULT_ERROR_PROGRAM_UNLINKED: - return "UR_RESULT_ERROR_PROGRAM_UNLINKED"; - case UR_RESULT_ERROR_OVERLAPPING_REGIONS: - return "UR_RESULT_ERROR_OVERLAPPING_REGIONS"; - case UR_RESULT_ERROR_INVALID_HOST_PTR: - return "UR_RESULT_ERROR_INVALID_HOST_PTR"; - case UR_RESULT_ERROR_INVALID_USM_SIZE: - return "UR_RESULT_ERROR_INVALID_USM_SIZE"; - case UR_RESULT_ERROR_OBJECT_ALLOCATION_FAILURE: - return "UR_RESULT_ERROR_OBJECT_ALLOCATION_FAILURE"; - case UR_RESULT_ERROR_ADAPTER_SPECIFIC: - return "UR_RESULT_ERROR_ADAPTER_SPECIFIC"; - default: - return "UR_RESULT_ERROR_UNKNOWN"; - } -}; - // Trace an internal PI call; returns in case of an error. #define UR_CALL(Call) \ { \ @@ -219,8 +77,7 @@ static auto getUrResultString = [](ur_result_t Result) { context.logger.debug("UR ---> {}", #Call); \ ur_result_t Result = (Call); \ if (PrintTrace) \ - context.logger.debug("UR <--- {}({})", #Call, \ - getUrResultString(Result)); \ + context.logger.debug("UR <--- {}({})", #Call, Result); \ if (Result != UR_RESULT_SUCCESS) \ return Result; \ } diff --git a/source/loader/layers/sanitizer/ur_sanitizer_layer.cpp b/source/loader/layers/sanitizer/ur_sanitizer_layer.cpp index 84a064997d..60d01ed1e6 100644 --- a/source/loader/layers/sanitizer/ur_sanitizer_layer.cpp +++ b/source/loader/layers/sanitizer/ur_sanitizer_layer.cpp @@ -18,7 +18,7 @@ context_t context; /////////////////////////////////////////////////////////////////////////////// context_t::context_t() - : interceptor(new SanitizerInterceptor()), + : interceptor(std::make_unique()), logger(logger::create_logger("sanitizer")) {} bool context_t::isAvailable() const { return true; } diff --git a/source/loader/layers/sanitizer/ur_sanitizer_layer.hpp b/source/loader/layers/sanitizer/ur_sanitizer_layer.hpp index 7e191b3785..8b9d961b62 100644 --- a/source/loader/layers/sanitizer/ur_sanitizer_layer.hpp +++ b/source/loader/layers/sanitizer/ur_sanitizer_layer.hpp @@ -32,7 +32,7 @@ enum class SanitizerType { class __urdlllocal context_t : public proxy_layer_context_t { public: ur_dditable_t urDdiTable = {}; - SanitizerInterceptor *interceptor = {}; + std::unique_ptr interceptor; logger::Logger logger; SanitizerType enabledType = SanitizerType::None; From 3b819f65665165a7e53bdf72439064c98d27931b Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Mon, 20 Nov 2023 21:18:32 -0800 Subject: [PATCH 50/72] resolve comments --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 27126a1340..70ac2863ae 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ List of options provided by CMake: | UR_USE_UBSAN | Enable UndefinedBehavior Sanitizer | ON/OFF | OFF | | UR_USE_MSAN | Enable MemorySanitizer (clang only) | ON/OFF | OFF | | UR_ENABLE_TRACING | Enable XPTI-based tracing layer | ON/OFF | OFF | -| UR_ENABLE_SANITIZER | Enable device sanitizer layer | ON/OFF | OFF | +| UR_ENABLE_SANITIZER | Enable device sanitizer layer | ON/OFF | ON | | UR_CONFORMANCE_TARGET_TRIPLES | SYCL triples to build CTS device binaries for | Comma-separated list | spir64 | | UR_BUILD_ADAPTER_L0 | Fetch and use level-zero adapter from SYCL | ON/OFF | OFF | | UR_BUILD_ADAPTER_OPENCL | Fetch and use opencl adapter from SYCL | ON/OFF | OFF | From 54220e93c1028c9cf95ccad7d112512c51fca323 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Mon, 20 Nov 2023 22:50:44 -0800 Subject: [PATCH 51/72] resolve comments --- .../layers/sanitizer/asan_interceptor.cpp | 203 ++++++++++-------- .../layers/sanitizer/asan_interceptor.hpp | 33 +-- source/loader/layers/sanitizer/ur_sanddi.cpp | 41 ++++ 3 files changed, 170 insertions(+), 107 deletions(-) diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index 45745be403..44c18b1f88 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -91,11 +91,11 @@ ur_result_t SanitizerInterceptor::allocateMemory( auto Alignment = Properties->align; assert(Alignment == 0 || IsPowerOfTwo(Alignment)); - auto &ContextInfo = getContextInfo(Context); - auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); + auto ContextInfo = getContextInfo(Context); + auto DeviceInfo = ContextInfo->getDeviceInfo(Device); if (Alignment == 0) { - Alignment = DeviceInfo.Alignment; + Alignment = DeviceInfo->Alignment; } // Copy from LLVM compiler-rt/lib/asan @@ -131,19 +131,19 @@ ur_result_t SanitizerInterceptor::allocateMemory( *ResultPtr = reinterpret_cast(UserBegin); - auto MemoryInfo = - USMAllocInfo{AllocBegin, UserBegin, UserEnd, NeededSize, Type}; + auto AllocInfo = std::make_shared( + USMAllocInfo{AllocBegin, UserBegin, UserEnd, NeededSize, Type}); // For updating shadow memory { - std::scoped_lock Guard(DeviceInfo.Mutex); - DeviceInfo.AllocInfos.emplace_back(MemoryInfo); + std::scoped_lock Guard(DeviceInfo->Mutex); + DeviceInfo->AllocInfos.emplace_back(AllocInfo); } // For memory release { - std::scoped_lock Guard(ContextInfo.Mutex); - ContextInfo.AllocatedUSMMap[AllocBegin] = MemoryInfo; + std::scoped_lock Guard(ContextInfo->Mutex); + ContextInfo->AllocatedUSMMap[AllocBegin] = AllocInfo; } context.logger.info( @@ -155,14 +155,14 @@ ur_result_t SanitizerInterceptor::allocateMemory( ur_result_t SanitizerInterceptor::releaseMemory(ur_context_handle_t Context, void *Ptr) { - auto &ContextInfo = getContextInfo(Context); + auto ContextInfo = getContextInfo(Context); - std::shared_lock Guard(ContextInfo.Mutex); + std::shared_lock Guard(ContextInfo->Mutex); auto Addr = (uptr)Ptr; // Find the last element is not greater than key - auto AllocInfoIt = ContextInfo.AllocatedUSMMap.upper_bound((uptr)Addr); - if (AllocInfoIt == ContextInfo.AllocatedUSMMap.begin()) { + auto AllocInfoIt = ContextInfo->AllocatedUSMMap.upper_bound((uptr)Addr); + if (AllocInfoIt == ContextInfo->AllocatedUSMMap.begin()) { context.logger.error( "Can't find release pointer({}) in AllocatedAddressesMap", Ptr); return UR_RESULT_ERROR_INVALID_ARGUMENT; @@ -171,17 +171,17 @@ ur_result_t SanitizerInterceptor::releaseMemory(ur_context_handle_t Context, auto &AllocInfo = AllocInfoIt->second; context.logger.debug("USMAllocInfo(AllocBegin={}, UserBegin={})", - AllocInfo.AllocBegin, AllocInfo.UserBegin); + AllocInfo->AllocBegin, AllocInfo->UserBegin); - if (Addr != AllocInfo.UserBegin) { + if (Addr != AllocInfo->UserBegin) { context.logger.error("Releasing pointer({}) is not match to {}", Ptr, - AllocInfo.UserBegin); + AllocInfo->UserBegin); return UR_RESULT_ERROR_INVALID_ARGUMENT; } // TODO: Update shadow memory return context.urDdiTable.USM.pfnFree(Context, - (void *)AllocInfo.AllocBegin); + (void *)AllocInfo->AllocBegin); } bool SanitizerInterceptor::preLaunchKernel(ur_kernel_handle_t Kernel, @@ -193,12 +193,12 @@ bool SanitizerInterceptor::preLaunchKernel(ur_kernel_handle_t Kernel, // Return LastEvent in QueueInfo auto Context = getContext(Queue); - auto &ContextInfo = getContextInfo(Context); - auto &QueueInfo = ContextInfo.getQueueInfo(Queue); + auto ContextInfo = getContextInfo(Context); + auto QueueInfo = ContextInfo->getQueueInfo(Queue); - std::scoped_lock Guard(QueueInfo.Mutex); - Event = QueueInfo.LastEvent; - QueueInfo.LastEvent = nullptr; + std::scoped_lock Guard(QueueInfo->Mutex); + Event = QueueInfo->LastEvent; + QueueInfo->LastEvent = nullptr; return true; } @@ -260,12 +260,12 @@ std::string SanitizerInterceptor::getKernelName(ur_kernel_handle_t Kernel) { return std::string(KernelNameBuf.data(), KernelNameSize); } -ur_result_t SanitizerInterceptor::allocShadowMemory(ur_context_handle_t Context, - DeviceInfo &DeviceInfo) { - if (DeviceInfo.Type == DeviceType::CPU) { - DeviceInfo.ShadowOffset = 0x00007fff7fffULL; - DeviceInfo.ShadowOffsetEnd = 0x10007fff7fffULL; - } else if (DeviceInfo.Type == DeviceType::GPU_PVC) { +ur_result_t SanitizerInterceptor::allocShadowMemory( + ur_context_handle_t Context, std::shared_ptr &DeviceInfo) { + if (DeviceInfo->Type == DeviceType::CPU) { + DeviceInfo->ShadowOffset = 0x00007fff7fffULL; + DeviceInfo->ShadowOffsetEnd = 0x10007fff7fffULL; + } else if (DeviceInfo->Type == DeviceType::GPU_PVC) { /// SHADOW MEMORY MAPPING (PVC, with CPU 47bit) /// Host/Shared USM : 0x0 ~ 0x0fff_ffff_ffff /// ? : 0x1000_0000_0000 ~ 0x1fff_ffff_ffff @@ -274,15 +274,15 @@ ur_result_t SanitizerInterceptor::allocShadowMemory(ur_context_handle_t Context, // TODO: Protect Bad Zone UR_CALL(context.urDdiTable.VirtualMem.pfnReserve( - Context, nullptr, SHADOW_SIZE, (void **)&DeviceInfo.ShadowOffset)); + Context, nullptr, SHADOW_SIZE, (void **)&DeviceInfo->ShadowOffset)); - DeviceInfo.ShadowOffsetEnd = DeviceInfo.ShadowOffset + SHADOW_SIZE; + DeviceInfo->ShadowOffsetEnd = DeviceInfo->ShadowOffset + SHADOW_SIZE; } else { die("Unsupport device type"); } context.logger.info("Device ShadowOffset: {} - {}", - (void *)DeviceInfo.ShadowOffset, - (void *)DeviceInfo.ShadowOffsetEnd); + (void *)DeviceInfo->ShadowOffset, + (void *)DeviceInfo->ShadowOffsetEnd); return UR_RESULT_SUCCESS; } @@ -294,18 +294,18 @@ ur_result_t SanitizerInterceptor::enqueueMemSetShadow( uint32_t NumEventsInWaitList = DepEvent ? 1 : 0; const ur_event_handle_t *EventsWaitList = DepEvent ? &DepEvent : nullptr; - auto &ContextInfo = getContextInfo(Context); - auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); + auto ContextInfo = getContextInfo(Context); + auto DeviceInfo = ContextInfo->getDeviceInfo(Device); - if (DeviceInfo.Type == DeviceType::CPU) { + if (DeviceInfo->Type == DeviceType::CPU) { die("Unsupport device type"); - } else if (DeviceInfo.Type == DeviceType::GPU_PVC) { + } else if (DeviceInfo->Type == DeviceType::GPU_PVC) { ur_event_handle_t InternalEvent{}; ur_event_handle_t *Event = OutEvent ? OutEvent : &InternalEvent; - uptr ShadowBegin = MemToShadow_PVC(DeviceInfo.ShadowOffset, Ptr); + uptr ShadowBegin = MemToShadow_PVC(DeviceInfo->ShadowOffset, Ptr); uptr ShadowEnd = - MemToShadow_PVC(DeviceInfo.ShadowOffset, Ptr + Size - 1); + MemToShadow_PVC(DeviceInfo->ShadowOffset, Ptr + Size - 1); // Maybe in future, we needn't to map physical memory manually const bool IsNeedMapPhysicalMem = true; @@ -380,25 +380,26 @@ ur_result_t SanitizerInterceptor::enqueueMemSetShadow( ur_result_t SanitizerInterceptor::enqueueAllocInfo( ur_context_handle_t Context, ur_device_handle_t Device, - ur_queue_handle_t Queue, USMAllocInfo &AllocInfo, + ur_queue_handle_t Queue, std::shared_ptr &AllocInfo, ur_event_handle_t &LastEvent) { // Init zero - UR_CALL(enqueueMemSetShadow(Context, Device, Queue, AllocInfo.AllocBegin, - AllocInfo.AllocSize, 0, LastEvent, &LastEvent)); + UR_CALL(enqueueMemSetShadow(Context, Device, Queue, AllocInfo->AllocBegin, + AllocInfo->AllocSize, 0, LastEvent, + &LastEvent)); - uptr TailBegin = RoundUpTo(AllocInfo.UserEnd, ASAN_SHADOW_GRANULARITY); - uptr TailEnd = AllocInfo.AllocBegin + AllocInfo.AllocSize; + uptr TailBegin = RoundUpTo(AllocInfo->UserEnd, ASAN_SHADOW_GRANULARITY); + uptr TailEnd = AllocInfo->AllocBegin + AllocInfo->AllocSize; // User tail - if (TailBegin != AllocInfo.UserEnd) { - auto Value = AllocInfo.UserEnd - - RoundDownTo(AllocInfo.UserEnd, ASAN_SHADOW_GRANULARITY); - UR_CALL(enqueueMemSetShadow(Context, Device, Queue, AllocInfo.UserEnd, + if (TailBegin != AllocInfo->UserEnd) { + auto Value = AllocInfo->UserEnd - + RoundDownTo(AllocInfo->UserEnd, ASAN_SHADOW_GRANULARITY); + UR_CALL(enqueueMemSetShadow(Context, Device, Queue, AllocInfo->UserEnd, 1, Value, LastEvent, &LastEvent)); } int ShadowByte; - switch (AllocInfo.Type) { + switch (AllocInfo->Type) { case USMMemoryType::HOST: ShadowByte = kUsmHostRedzoneMagic; break; @@ -413,12 +414,12 @@ ur_result_t SanitizerInterceptor::enqueueAllocInfo( break; default: ShadowByte = 0xff; - assert(false && "Unknow AllocInfo.Type"); + assert(false && "Unknow AllocInfo Type"); } // Left red zone - UR_CALL(enqueueMemSetShadow(Context, Device, Queue, AllocInfo.AllocBegin, - AllocInfo.UserBegin - AllocInfo.AllocBegin, + UR_CALL(enqueueMemSetShadow(Context, Device, Queue, AllocInfo->AllocBegin, + AllocInfo->UserBegin - AllocInfo->AllocBegin, ShadowByte, LastEvent, &LastEvent)); // Right red zone @@ -434,61 +435,68 @@ ur_result_t SanitizerInterceptor::updateShadowMemory(ur_queue_handle_t Queue) { auto Device = getDevice(Queue); assert(Device != nullptr); - auto &ContextInfo = getContextInfo(Context); + auto ContextInfo = getContextInfo(Context); - auto &HostInfo = ContextInfo.getDeviceInfo(nullptr); - auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); - auto &QueueInfo = ContextInfo.getQueueInfo(Queue); + auto HostInfo = ContextInfo->getDeviceInfo(nullptr); + auto DeviceInfo = ContextInfo->getDeviceInfo(Device); + auto QueueInfo = ContextInfo->getQueueInfo(Queue); - std::shared_lock HostGuard(HostInfo.Mutex, + std::shared_lock HostGuard(HostInfo->Mutex, std::defer_lock); - std::unique_lock DeviceGuard(DeviceInfo.Mutex, + std::unique_lock DeviceGuard(DeviceInfo->Mutex, std::defer_lock); std::scoped_lock, std::unique_lock, ur_mutex> - Guard(HostGuard, DeviceGuard, QueueInfo.Mutex); + Guard(HostGuard, DeviceGuard, QueueInfo->Mutex); - ur_event_handle_t LastEvent = QueueInfo.LastEvent; + ur_event_handle_t LastEvent = QueueInfo->LastEvent; // FIXME: Always update host USM, but it'd be better to update host USM // selectively, or each devices once - for (auto &AllocInfo : HostInfo.AllocInfos) { + for (auto &AllocInfo : HostInfo->AllocInfos) { UR_CALL(enqueueAllocInfo(Context, Device, Queue, AllocInfo, LastEvent)); } - for (auto &AllocInfo : DeviceInfo.AllocInfos) { + for (auto &AllocInfo : DeviceInfo->AllocInfos) { UR_CALL(enqueueAllocInfo(Context, Device, Queue, AllocInfo, LastEvent)); } - DeviceInfo.AllocInfos.clear(); + DeviceInfo->AllocInfos.clear(); - QueueInfo.LastEvent = LastEvent; + QueueInfo->LastEvent = LastEvent; return UR_RESULT_SUCCESS; } ur_result_t SanitizerInterceptor::addContext(ur_context_handle_t Context) { - auto ContextInfoPtr = std::make_unique(); + auto ContextInfo = std::make_shared(); // Host Device - auto DeviceInfoPtr = std::make_unique(); - DeviceInfoPtr->Type = DeviceType::CPU; - DeviceInfoPtr->Alignment = ASAN_SHADOW_GRANULARITY; + auto DeviceInfo = std::make_shared(); + DeviceInfo->Type = DeviceType::CPU; + DeviceInfo->Alignment = ASAN_SHADOW_GRANULARITY; // TODO: Check if host asan is enabled - DeviceInfoPtr->ShadowOffset = 0; - DeviceInfoPtr->ShadowOffsetEnd = 0; + DeviceInfo->ShadowOffset = 0; + DeviceInfo->ShadowOffsetEnd = 0; - ContextInfoPtr->DeviceMap.emplace(nullptr, std::move(DeviceInfoPtr)); + ContextInfo->DeviceMap.emplace(nullptr, std::move(DeviceInfo)); std::scoped_lock Guard(m_ContextMapMutex); - m_ContextMap.emplace(Context, std::move(ContextInfoPtr)); + m_ContextMap.emplace(Context, std::move(ContextInfo)); + + return UR_RESULT_SUCCESS; +} +ur_result_t SanitizerInterceptor::removeContext(ur_context_handle_t Context) { + std::scoped_lock Guard(m_ContextMapMutex); + assert(m_ContextMap.find(Context) != m_ContextMap.end()); + m_ContextMap.erase(Context); return UR_RESULT_SUCCESS; } ur_result_t SanitizerInterceptor::addDevice(ur_context_handle_t Context, ur_device_handle_t Device) { - auto DeviceInfoPtr = std::make_unique(); + auto DeviceInfo = std::make_shared(); // Query device type ur_device_type_t DeviceType; @@ -496,39 +504,48 @@ ur_result_t SanitizerInterceptor::addDevice(ur_context_handle_t Context, Device, UR_DEVICE_INFO_TYPE, sizeof(DeviceType), &DeviceType, nullptr)); switch (DeviceType) { case UR_DEVICE_TYPE_CPU: - DeviceInfoPtr->Type = DeviceType::CPU; + DeviceInfo->Type = DeviceType::CPU; break; case UR_DEVICE_TYPE_GPU: - DeviceInfoPtr->Type = DeviceType::GPU_PVC; + DeviceInfo->Type = DeviceType::GPU_PVC; break; default: - DeviceInfoPtr->Type = DeviceType::UNKNOWN; + DeviceInfo->Type = DeviceType::UNKNOWN; } // Query alignment UR_CALL(context.urDdiTable.Device.pfnGetInfo( Device, UR_DEVICE_INFO_MEM_BASE_ADDR_ALIGN, - sizeof(DeviceInfoPtr->Alignment), &DeviceInfoPtr->Alignment, nullptr)); + sizeof(DeviceInfo->Alignment), &DeviceInfo->Alignment, nullptr)); // Allocate shadow memory - UR_CALL(allocShadowMemory(Context, *DeviceInfoPtr.get())); + UR_CALL(allocShadowMemory(Context, DeviceInfo)); - auto &ContextInfo = getContextInfo(Context); - std::scoped_lock Guard(ContextInfo.Mutex); - ContextInfo.DeviceMap.emplace(Device, std::move(DeviceInfoPtr)); + auto ContextInfo = getContextInfo(Context); + std::scoped_lock Guard(ContextInfo->Mutex); + ContextInfo->DeviceMap.emplace(Device, std::move(DeviceInfo)); return UR_RESULT_SUCCESS; } ur_result_t SanitizerInterceptor::addQueue(ur_context_handle_t Context, ur_queue_handle_t Queue) { - auto QueueInfoPtr = std::make_unique(); - QueueInfoPtr->LastEvent = nullptr; + auto QueueInfo = std::make_shared(); + QueueInfo->LastEvent = nullptr; + + auto ContextInfo = getContextInfo(Context); + std::scoped_lock Guard(ContextInfo->Mutex); + ContextInfo->QueueMap.emplace(Queue, std::move(QueueInfo)); - auto &ContextInfo = getContextInfo(Context); - std::scoped_lock Guard(ContextInfo.Mutex); - ContextInfo.QueueMap.emplace(Queue, std::move(QueueInfoPtr)); + return UR_RESULT_SUCCESS; +} +ur_result_t SanitizerInterceptor::removeQueue(ur_context_handle_t Context, + ur_queue_handle_t Queue) { + auto ContextInfo = getContextInfo(Context); + std::scoped_lock Guard(ContextInfo->Mutex); + assert(ContextInfo->QueueMap.find(Queue) != ContextInfo->QueueMap.end()); + ContextInfo->QueueMap.erase(Queue); return UR_RESULT_SUCCESS; } @@ -538,12 +555,12 @@ void SanitizerInterceptor::prepareLaunch(ur_queue_handle_t Queue, auto Device = getDevice(Queue); auto Program = getProgram(Kernel); - auto &ContextInfo = getContextInfo(Context); - auto &DeviceInfo = ContextInfo.getDeviceInfo(Device); - auto &QueueInfo = ContextInfo.getQueueInfo(Queue); + auto ContextInfo = getContextInfo(Context); + auto DeviceInfo = ContextInfo->getDeviceInfo(Device); + auto QueueInfo = ContextInfo->getQueueInfo(Queue); - std::scoped_lock Guard(QueueInfo.Mutex); - ur_event_handle_t LastEvent = QueueInfo.LastEvent; + std::scoped_lock Guard(QueueInfo->Mutex); + ur_event_handle_t LastEvent = QueueInfo->LastEvent; do { // Set global variable to program @@ -567,12 +584,12 @@ void SanitizerInterceptor::prepareLaunch(ur_queue_handle_t Queue, // Device shadow memory offset EnqueueWriteGlobal(kSPIR_AsanShadowMemoryGlobalStart, - &DeviceInfo.ShadowOffset); + &DeviceInfo->ShadowOffset); EnqueueWriteGlobal(kSPIR_AsanShadowMemoryGlobalEnd, - &DeviceInfo.ShadowOffsetEnd); + &DeviceInfo->ShadowOffsetEnd); } while (false); - QueueInfo.LastEvent = LastEvent; + QueueInfo->LastEvent = LastEvent; } } // namespace ur_sanitizer_layer diff --git a/source/loader/layers/sanitizer/asan_interceptor.hpp b/source/loader/layers/sanitizer/asan_interceptor.hpp index e83378deae..8b61865b53 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.hpp +++ b/source/loader/layers/sanitizer/asan_interceptor.hpp @@ -41,7 +41,7 @@ struct DeviceInfo { // Lock InitPool & AllocInfos ur_shared_mutex Mutex; - std::vector AllocInfos; + std::vector> AllocInfos; }; struct QueueInfo { @@ -51,19 +51,19 @@ struct QueueInfo { struct ContextInfo { - DeviceInfo &getDeviceInfo(ur_device_handle_t Device) { + std::shared_ptr getDeviceInfo(ur_device_handle_t Device) { std::shared_lock Guard(Mutex); assert(DeviceMap.find(Device) != DeviceMap.end()); - return *DeviceMap[Device].get(); + return DeviceMap[Device]; } - QueueInfo &getQueueInfo(ur_queue_handle_t Queue) { + std::shared_ptr getQueueInfo(ur_queue_handle_t Queue) { std::shared_lock Guard(Mutex); assert(QueueMap.find(Queue) != QueueMap.end()); - return *QueueMap[Queue].get(); + return QueueMap[Queue]; } - USMAllocInfo &getUSMAllocInfo(uptr Address) { + std::shared_ptr getUSMAllocInfo(uptr Address) { std::shared_lock Guard(Mutex); assert(AllocatedUSMMap.find(Address) != AllocatedUSMMap.end()); return AllocatedUSMMap[Address]; @@ -71,14 +71,14 @@ struct ContextInfo { ur_shared_mutex Mutex; // Note: nullptr is host device - std::unordered_map> + std::unordered_map> DeviceMap; - std::unordered_map> QueueMap; + std::unordered_map> QueueMap; /// key: USMAllocInfo.AllocBegin /// value: USMAllocInfo /// Use AllocBegin as key can help to detect underflow pointer - std::map AllocatedUSMMap; + std::map> AllocatedUSMMap; }; class SanitizerInterceptor { @@ -98,16 +98,21 @@ class SanitizerInterceptor { ur_event_handle_t *Event, bool SetCallback = true); ur_result_t addContext(ur_context_handle_t Context); + ur_result_t removeContext(ur_context_handle_t Context); + ur_result_t addDevice(ur_context_handle_t Context, ur_device_handle_t Device); + ur_result_t addQueue(ur_context_handle_t Context, ur_queue_handle_t Queue); + ur_result_t removeQueue(ur_context_handle_t Context, + ur_queue_handle_t Queue); private: ur_result_t updateShadowMemory(ur_queue_handle_t Queue); ur_result_t enqueueAllocInfo(ur_context_handle_t Context, ur_device_handle_t Device, ur_queue_handle_t Queue, - USMAllocInfo &AlloccInfo, + std::shared_ptr &AlloccInfo, ur_event_handle_t &LastEvent); /// Initialize Global Variables & Kernel Name at first Launch @@ -115,7 +120,7 @@ class SanitizerInterceptor { std::string getKernelName(ur_kernel_handle_t Kernel); ur_result_t allocShadowMemory(ur_context_handle_t Context, - DeviceInfo &DeviceInfo); + std::shared_ptr &DeviceInfo); ur_result_t enqueueMemSetShadow(ur_context_handle_t Context, ur_device_handle_t Device, ur_queue_handle_t Queue, uptr Addr, @@ -123,14 +128,14 @@ class SanitizerInterceptor { ur_event_handle_t DepEvent, ur_event_handle_t *OutEvent); - ContextInfo &getContextInfo(ur_context_handle_t Context) { + std::shared_ptr getContextInfo(ur_context_handle_t Context) { std::shared_lock Guard(m_ContextMapMutex); assert(m_ContextMap.find(Context) != m_ContextMap.end()); - return *m_ContextMap[Context].get(); + return m_ContextMap[Context]; } private: - std::unordered_map> + std::unordered_map> m_ContextMap; ur_shared_mutex m_ContextMapMutex; }; diff --git a/source/loader/layers/sanitizer/ur_sanddi.cpp b/source/loader/layers/sanitizer/ur_sanddi.cpp index 40af911cc9..3e0071cddd 100644 --- a/source/loader/layers/sanitizer/ur_sanddi.cpp +++ b/source/loader/layers/sanitizer/ur_sanddi.cpp @@ -122,6 +122,28 @@ __urdlllocal ur_result_t UR_APICALL urQueueCreate( return result; } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urQueueRelease +__urdlllocal ur_result_t UR_APICALL urQueueRelease( + ur_queue_handle_t hQueue ///< [in] handle of the queue object to release +) { + auto pfnRelease = context.urDdiTable.Queue.pfnRelease; + + if (nullptr == pfnRelease) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + + ur_context_handle_t hContext; + UR_CALL(context.urDdiTable.Queue.pfnGetInfo(hQueue, UR_QUEUE_INFO_CONTEXT, + sizeof(ur_context_handle_t), + &hContext, nullptr)); + UR_CALL(context.interceptor->removeQueue(hContext, hQueue)); + + ur_result_t result = pfnRelease(hQueue); + + return result; +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Intercept function for urEnqueueKernelLaunch __urdlllocal ur_result_t UR_APICALL urEnqueueKernelLaunch( @@ -259,6 +281,23 @@ __urdlllocal ur_result_t UR_APICALL urContextCreateWithNativeHandle( return result; } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urContextRelease +__urdlllocal ur_result_t UR_APICALL urContextRelease( + ur_context_handle_t hContext ///< [in] handle of the context to release. +) { + auto pfnRelease = context.urDdiTable.Context.pfnRelease; + + if (nullptr == pfnRelease) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + + UR_CALL(context.interceptor->removeContext(hContext)); + ur_result_t result = pfnRelease(hContext); + + return result; +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Exported function for filling application's Context table /// with current process' addresses @@ -286,6 +325,7 @@ __urdlllocal ur_result_t UR_APICALL urGetContextProcAddrTable( ur_result_t result = UR_RESULT_SUCCESS; pDdiTable->pfnCreate = ur_sanitizer_layer::urContextCreate; + pDdiTable->pfnRelease = ur_sanitizer_layer::urContextRelease; pDdiTable->pfnCreateWithNativeHandle = ur_sanitizer_layer::urContextCreateWithNativeHandle; @@ -349,6 +389,7 @@ __urdlllocal ur_result_t UR_APICALL urGetQueueProcAddrTable( ur_result_t result = UR_RESULT_SUCCESS; pDdiTable->pfnCreate = ur_sanitizer_layer::urQueueCreate; + pDdiTable->pfnRelease = ur_sanitizer_layer::urQueueRelease; return result; } From e59d7c7f3f2ffa652b37ff7357be4a9167508c83 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Thu, 7 Dec 2023 18:47:05 -0800 Subject: [PATCH 52/72] fix build errors --- source/loader/layers/sanitizer/asan_interceptor.cpp | 3 +-- source/loader/layers/sanitizer/asan_interceptor.hpp | 2 +- source/loader/layers/sanitizer/common.hpp | 1 - source/loader/layers/sanitizer/ur_sanddi.cpp | 5 +++-- source/loader/layers/sanitizer/ur_sanitizer_layer.cpp | 2 ++ source/loader/layers/sanitizer/ur_sanitizer_layer.hpp | 5 ++++- 6 files changed, 11 insertions(+), 7 deletions(-) diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index 44c18b1f88..8b21e38641 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -205,8 +205,7 @@ bool SanitizerInterceptor::preLaunchKernel(ur_kernel_handle_t Kernel, void SanitizerInterceptor::postLaunchKernel(ur_kernel_handle_t Kernel, ur_queue_handle_t Queue, - ur_event_handle_t *Event, - bool SetCallback) { + ur_event_handle_t *Event) { auto Program = getProgram(Kernel); ur_event_handle_t ReadEvent{}; diff --git a/source/loader/layers/sanitizer/asan_interceptor.hpp b/source/loader/layers/sanitizer/asan_interceptor.hpp index 8b61865b53..d2afc7d0c8 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.hpp +++ b/source/loader/layers/sanitizer/asan_interceptor.hpp @@ -95,7 +95,7 @@ class SanitizerInterceptor { bool preLaunchKernel(ur_kernel_handle_t Kernel, ur_queue_handle_t Queue, ur_event_handle_t &Event); void postLaunchKernel(ur_kernel_handle_t Kernel, ur_queue_handle_t Queue, - ur_event_handle_t *Event, bool SetCallback = true); + ur_event_handle_t *Event); ur_result_t addContext(ur_context_handle_t Context); ur_result_t removeContext(ur_context_handle_t Context); diff --git a/source/loader/layers/sanitizer/common.hpp b/source/loader/layers/sanitizer/common.hpp index 0a783dbd5d..2974b0c3e7 100644 --- a/source/loader/layers/sanitizer/common.hpp +++ b/source/loader/layers/sanitizer/common.hpp @@ -14,7 +14,6 @@ #include "ur/ur.hpp" #include "ur_ddi.h" -#include "ur_params.hpp" #include #include diff --git a/source/loader/layers/sanitizer/ur_sanddi.cpp b/source/loader/layers/sanitizer/ur_sanddi.cpp index 3e0071cddd..3c2be0cd7f 100644 --- a/source/loader/layers/sanitizer/ur_sanddi.cpp +++ b/source/loader/layers/sanitizer/ur_sanddi.cpp @@ -198,7 +198,7 @@ __urdlllocal ur_result_t UR_APICALL urEnqueueKernelLaunch( pLocalWorkSize, numEventsInWaitList, phEventWaitList, phEvent); if (result == UR_RESULT_SUCCESS) { - context.interceptor->postLaunchKernel(hKernel, hQueue, phEvent, false); + context.interceptor->postLaunchKernel(hKernel, hQueue, phEvent); } return result; @@ -425,7 +425,8 @@ __urdlllocal ur_result_t UR_APICALL urGetUSMProcAddrTable( } ur_result_t context_t::init(ur_dditable_t *dditable, - const std::set &enabledLayerNames) { + const std::set &enabledLayerNames, + [[maybe_unused]] codeloc_data codelocData) { ur_result_t result = UR_RESULT_SUCCESS; if (enabledLayerNames.count("UR_LAYER_ASAN")) { diff --git a/source/loader/layers/sanitizer/ur_sanitizer_layer.cpp b/source/loader/layers/sanitizer/ur_sanitizer_layer.cpp index 60d01ed1e6..0df123b6c2 100644 --- a/source/loader/layers/sanitizer/ur_sanitizer_layer.cpp +++ b/source/loader/layers/sanitizer/ur_sanitizer_layer.cpp @@ -23,6 +23,8 @@ context_t::context_t() bool context_t::isAvailable() const { return true; } +ur_result_t context_t::tearDown() { return UR_RESULT_SUCCESS; } + /////////////////////////////////////////////////////////////////////////////// context_t::~context_t() {} } // namespace ur_sanitizer_layer diff --git a/source/loader/layers/sanitizer/ur_sanitizer_layer.hpp b/source/loader/layers/sanitizer/ur_sanitizer_layer.hpp index 8b9d961b62..018d9f4a80 100644 --- a/source/loader/layers/sanitizer/ur_sanitizer_layer.hpp +++ b/source/loader/layers/sanitizer/ur_sanitizer_layer.hpp @@ -45,7 +45,10 @@ class __urdlllocal context_t : public proxy_layer_context_t { return {"UR_LAYER_ASAN", "UR_LAYER_MSAN", "UR_LAYER_TSAN"}; } ur_result_t init(ur_dditable_t *dditable, - const std::set &enabledLayerNames) override; + const std::set &enabledLayerNames, + codeloc_data codelocData) override; + + ur_result_t tearDown() override; }; extern context_t context; From bba6f82bbeb7595401477f0cc20da7c898808e3f Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Fri, 8 Dec 2023 00:20:48 -0800 Subject: [PATCH 53/72] add basic test --- test/layers/CMakeLists.txt | 4 +++ test/layers/sanitizer/CMakeLists.txt | 35 +++++++++++++++++++++++ test/layers/sanitizer/test_asan.cpp | 24 ++++++++++++++++ test/layers/sanitizer/test_asan.out.match | 0 4 files changed, 63 insertions(+) create mode 100644 test/layers/sanitizer/CMakeLists.txt create mode 100644 test/layers/sanitizer/test_asan.cpp create mode 100644 test/layers/sanitizer/test_asan.out.match diff --git a/test/layers/CMakeLists.txt b/test/layers/CMakeLists.txt index 2c10a08518..fbf532c274 100644 --- a/test/layers/CMakeLists.txt +++ b/test/layers/CMakeLists.txt @@ -8,3 +8,7 @@ add_subdirectory(validation) if(UR_ENABLE_TRACING) add_subdirectory(tracing) endif() + +if(UR_ENABLE_SANITIZER) + add_subdirectory(sanitizer) +endif() diff --git a/test/layers/sanitizer/CMakeLists.txt b/test/layers/sanitizer/CMakeLists.txt new file mode 100644 index 0000000000..c9d725c9b2 --- /dev/null +++ b/test/layers/sanitizer/CMakeLists.txt @@ -0,0 +1,35 @@ +# Copyright (C) 2023 Intel Corporation +# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +# See LICENSE.TXT +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +function(add_sanitizer_test name) + set(TEST_TARGET_NAME sanitizer_test-${name}) + add_ur_executable(${TEST_TARGET_NAME} + ${ARGN}) + target_link_libraries(${TEST_TARGET_NAME} + PRIVATE + ${PROJECT_NAME}::loader + ${PROJECT_NAME}::headers + ${PROJECT_NAME}::testing + GTest::gtest_main) + + add_test(NAME test_${name} + COMMAND ${CMAKE_COMMAND} + -D MODE=stderr + -D TEST_FILE=$ + -D MATCH_FILE=${CMAKE_CURRENT_SOURCE_DIR}/test_${name}.out.match + -P ${PROJECT_SOURCE_DIR}/cmake/match.cmake + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + + set_tests_properties(test_${name} PROPERTIES + LABELS ${name} + ) + + string(TOUPPER ${name} upper_name) + set_property(TEST test_${name} PROPERTY ENVIRONMENT + "UR_ADAPTERS_FORCE_LOAD=\"$\"") +endfunction() + +add_sanitizer_test(asan test_asan.cpp) diff --git a/test/layers/sanitizer/test_asan.cpp b/test/layers/sanitizer/test_asan.cpp new file mode 100644 index 0000000000..36f4d78271 --- /dev/null +++ b/test/layers/sanitizer/test_asan.cpp @@ -0,0 +1,24 @@ +/* + * + * Copyright (C) 2023 Intel Corporation + * + * Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. + * See LICENSE.TXT + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + * @file codeloc.cpp + * + */ + +#include +#include + +TEST(sanitizerTest, asan) { + ur_loader_config_handle_t loader_config; + ASSERT_EQ(urLoaderConfigCreate(&loader_config), UR_RESULT_SUCCESS); + + ASSERT_EQ(urLoaderConfigEnableLayer(loader_config, "UR_LAYER_ASAN"), + UR_RESULT_SUCCESS); + + ASSERT_EQ(urLoaderInit(0, loader_config), UR_RESULT_SUCCESS); +} diff --git a/test/layers/sanitizer/test_asan.out.match b/test/layers/sanitizer/test_asan.out.match new file mode 100644 index 0000000000..e69de29bb2 From a084650e235d24d5f0e9a6a1770ca46282e6cec6 Mon Sep 17 00:00:00 2001 From: Allan Zyne Date: Sat, 9 Dec 2023 10:04:53 +0800 Subject: [PATCH 54/72] fix warning & remove tests --- .../layers/sanitizer/asan_interceptor.cpp | 4 +-- test/layers/CMakeLists.txt | 4 --- test/layers/sanitizer/CMakeLists.txt | 35 ------------------- test/layers/sanitizer/test_asan.cpp | 24 ------------- test/layers/sanitizer/test_asan.out.match | 0 5 files changed, 2 insertions(+), 65 deletions(-) delete mode 100644 test/layers/sanitizer/CMakeLists.txt delete mode 100644 test/layers/sanitizer/test_asan.cpp delete mode 100644 test/layers/sanitizer/test_asan.out.match diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index 8b21e38641..0a47880639 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -231,8 +231,8 @@ void SanitizerInterceptor::postLaunchKernel(ur_kernel_handle_t Kernel, DeviceSanitizerFormat(AH->ErrorType), DeviceSanitizerFormat(AH->MemoryType)); fprintf(stderr, - "%s of size %u at kernel <%s> LID(%lu, %lu, %lu) GID(%lu, " - "%lu, %lu)\n", + "%s of size %u at kernel <%s> LID(%llu, %llu, %llu) GID(%llu, " + "%llu, %llu)\n", AH->IsWrite ? "WRITE" : "READ", AH->AccessSize, Func, AH->LID0, AH->LID1, AH->LID2, AH->GID0, AH->GID1, AH->GID2); fprintf(stderr, " #0 %s %s:%d\n", Func, File, AH->Line); diff --git a/test/layers/CMakeLists.txt b/test/layers/CMakeLists.txt index fbf532c274..2c10a08518 100644 --- a/test/layers/CMakeLists.txt +++ b/test/layers/CMakeLists.txt @@ -8,7 +8,3 @@ add_subdirectory(validation) if(UR_ENABLE_TRACING) add_subdirectory(tracing) endif() - -if(UR_ENABLE_SANITIZER) - add_subdirectory(sanitizer) -endif() diff --git a/test/layers/sanitizer/CMakeLists.txt b/test/layers/sanitizer/CMakeLists.txt deleted file mode 100644 index c9d725c9b2..0000000000 --- a/test/layers/sanitizer/CMakeLists.txt +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (C) 2023 Intel Corporation -# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. -# See LICENSE.TXT -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -function(add_sanitizer_test name) - set(TEST_TARGET_NAME sanitizer_test-${name}) - add_ur_executable(${TEST_TARGET_NAME} - ${ARGN}) - target_link_libraries(${TEST_TARGET_NAME} - PRIVATE - ${PROJECT_NAME}::loader - ${PROJECT_NAME}::headers - ${PROJECT_NAME}::testing - GTest::gtest_main) - - add_test(NAME test_${name} - COMMAND ${CMAKE_COMMAND} - -D MODE=stderr - -D TEST_FILE=$ - -D MATCH_FILE=${CMAKE_CURRENT_SOURCE_DIR}/test_${name}.out.match - -P ${PROJECT_SOURCE_DIR}/cmake/match.cmake - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - ) - - set_tests_properties(test_${name} PROPERTIES - LABELS ${name} - ) - - string(TOUPPER ${name} upper_name) - set_property(TEST test_${name} PROPERTY ENVIRONMENT - "UR_ADAPTERS_FORCE_LOAD=\"$\"") -endfunction() - -add_sanitizer_test(asan test_asan.cpp) diff --git a/test/layers/sanitizer/test_asan.cpp b/test/layers/sanitizer/test_asan.cpp deleted file mode 100644 index 36f4d78271..0000000000 --- a/test/layers/sanitizer/test_asan.cpp +++ /dev/null @@ -1,24 +0,0 @@ -/* - * - * Copyright (C) 2023 Intel Corporation - * - * Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. - * See LICENSE.TXT - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - * - * @file codeloc.cpp - * - */ - -#include -#include - -TEST(sanitizerTest, asan) { - ur_loader_config_handle_t loader_config; - ASSERT_EQ(urLoaderConfigCreate(&loader_config), UR_RESULT_SUCCESS); - - ASSERT_EQ(urLoaderConfigEnableLayer(loader_config, "UR_LAYER_ASAN"), - UR_RESULT_SUCCESS); - - ASSERT_EQ(urLoaderInit(0, loader_config), UR_RESULT_SUCCESS); -} diff --git a/test/layers/sanitizer/test_asan.out.match b/test/layers/sanitizer/test_asan.out.match deleted file mode 100644 index e69de29bb2..0000000000 From 1148ccbc5db5a752e1c05999dc265ecc829010da Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Sun, 10 Dec 2023 04:22:16 -0800 Subject: [PATCH 55/72] fix warning --- .../layers/sanitizer/asan_interceptor.cpp | 18 +++++++++--------- source/loader/layers/sanitizer/common.hpp | 6 ++++++ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index 0a47880639..30666f499d 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -57,7 +57,7 @@ uptr MemToShadow_PVC(uptr USM_SHADOW_BASE, uptr UPtr) { ur_context_handle_t getContext(ur_queue_handle_t Queue) { ur_context_handle_t Context; - auto Result = context.urDdiTable.Queue.pfnGetInfo( + [[maybe_unused]] auto Result = context.urDdiTable.Queue.pfnGetInfo( Queue, UR_QUEUE_INFO_CONTEXT, sizeof(ur_context_handle_t), &Context, nullptr); assert(Result == UR_RESULT_SUCCESS); @@ -66,7 +66,7 @@ ur_context_handle_t getContext(ur_queue_handle_t Queue) { ur_device_handle_t getDevice(ur_queue_handle_t Queue) { ur_device_handle_t Device; - auto Result = context.urDdiTable.Queue.pfnGetInfo( + [[maybe_unused]] auto Result = context.urDdiTable.Queue.pfnGetInfo( Queue, UR_QUEUE_INFO_DEVICE, sizeof(ur_device_handle_t), &Device, nullptr); assert(Result == UR_RESULT_SUCCESS); @@ -75,7 +75,7 @@ ur_device_handle_t getDevice(ur_queue_handle_t Queue) { ur_program_handle_t getProgram(ur_kernel_handle_t Kernel) { ur_program_handle_t Program; - auto Result = context.urDdiTable.Kernel.pfnGetInfo( + [[maybe_unused]] auto Result = context.urDdiTable.Kernel.pfnGetInfo( Kernel, UR_KERNEL_INFO_PROGRAM, sizeof(ur_program_handle_t), &Program, nullptr); assert(Result == UR_RESULT_SUCCESS); @@ -92,6 +92,7 @@ ur_result_t SanitizerInterceptor::allocateMemory( assert(Alignment == 0 || IsPowerOfTwo(Alignment)); auto ContextInfo = getContextInfo(Context); + // Device is nullptr if Type == USMMemoryType::HOST auto DeviceInfo = ContextInfo->getDeviceInfo(Device); if (Alignment == 0) { @@ -121,7 +122,7 @@ ur_result_t SanitizerInterceptor::allocateMemory( // Copy from LLVM compiler-rt/lib/asan uptr AllocBegin = reinterpret_cast(Allocated); - uptr AllocEnd = AllocBegin + NeededSize; + [[maybe_unused]] uptr AllocEnd = AllocBegin + NeededSize; uptr UserBegin = AllocBegin + RZSize; if (!IsAligned(UserBegin, Alignment)) { UserBegin = RoundUpTo(UserBegin, Alignment); @@ -245,7 +246,7 @@ void SanitizerInterceptor::postLaunchKernel(ur_kernel_handle_t Kernel, std::string SanitizerInterceptor::getKernelName(ur_kernel_handle_t Kernel) { size_t KernelNameSize = 0; - auto Res = context.urDdiTable.Kernel.pfnGetInfo( + [[maybe_unused]] auto Res = context.urDdiTable.Kernel.pfnGetInfo( Kernel, UR_KERNEL_INFO_FUNCTION_NAME, 0, nullptr, &KernelNameSize); assert(Res == UR_RESULT_SUCCESS); @@ -262,8 +263,7 @@ std::string SanitizerInterceptor::getKernelName(ur_kernel_handle_t Kernel) { ur_result_t SanitizerInterceptor::allocShadowMemory( ur_context_handle_t Context, std::shared_ptr &DeviceInfo) { if (DeviceInfo->Type == DeviceType::CPU) { - DeviceInfo->ShadowOffset = 0x00007fff7fffULL; - DeviceInfo->ShadowOffsetEnd = 0x10007fff7fffULL; + die("Unsupport device type"); } else if (DeviceInfo->Type == DeviceType::GPU_PVC) { /// SHADOW MEMORY MAPPING (PVC, with CPU 47bit) /// Host/Shared USM : 0x0 ~ 0x0fff_ffff_ffff @@ -561,7 +561,7 @@ void SanitizerInterceptor::prepareLaunch(ur_queue_handle_t Queue, std::scoped_lock Guard(QueueInfo->Mutex); ur_event_handle_t LastEvent = QueueInfo->LastEvent; - do { + { // Set global variable to program auto EnqueueWriteGlobal = [&](const char *Name, const void *Value) { ur_event_handle_t NewEvent{}; @@ -586,7 +586,7 @@ void SanitizerInterceptor::prepareLaunch(ur_queue_handle_t Queue, &DeviceInfo->ShadowOffset); EnqueueWriteGlobal(kSPIR_AsanShadowMemoryGlobalEnd, &DeviceInfo->ShadowOffsetEnd); - } while (false); + } QueueInfo->LastEvent = LastEvent; } diff --git a/source/loader/layers/sanitizer/common.hpp b/source/loader/layers/sanitizer/common.hpp index 2974b0c3e7..05ac3f0c07 100644 --- a/source/loader/layers/sanitizer/common.hpp +++ b/source/loader/layers/sanitizer/common.hpp @@ -81,4 +81,10 @@ inline constexpr uptr ComputeRZLog(uptr user_requested_size) { return Result; \ } +#ifndef NDEBUG +#define UR_ASSERT_EQ(Call, Result) assert(Call == Result) +#else +#define UR_ASSERT_EQ(Call, Result) (void)Call +#endif + } // namespace ur_sanitizer_layer From 524a83dc7b6132f33e2d66a0736120d686ebe334 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Sun, 10 Dec 2023 04:49:22 -0800 Subject: [PATCH 56/72] fix warning --- source/loader/layers/sanitizer/asan_interceptor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index 30666f499d..6640c46bc9 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -232,8 +232,8 @@ void SanitizerInterceptor::postLaunchKernel(ur_kernel_handle_t Kernel, DeviceSanitizerFormat(AH->ErrorType), DeviceSanitizerFormat(AH->MemoryType)); fprintf(stderr, - "%s of size %u at kernel <%s> LID(%llu, %llu, %llu) GID(%llu, " - "%llu, %llu)\n", + "%s of size %u at kernel <%s> LID(%lu, %lu, %lu) GID(%lu, " + "%lu, %lu)\n", AH->IsWrite ? "WRITE" : "READ", AH->AccessSize, Func, AH->LID0, AH->LID1, AH->LID2, AH->GID0, AH->GID1, AH->GID2); fprintf(stderr, " #0 %s %s:%d\n", Func, File, AH->Line); From e9a20938f0da23a1e63edbae43a8e54c61e05593 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Tue, 12 Dec 2023 02:00:57 -0800 Subject: [PATCH 57/72] fix fprintf --- source/common/logger/ur_logger.hpp | 5 +++++ source/common/logger/ur_logger_details.hpp | 7 +++++++ .../layers/sanitizer/asan_interceptor.cpp | 20 +++++++++---------- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/source/common/logger/ur_logger.hpp b/source/common/logger/ur_logger.hpp index 0575660fb6..27fcca4274 100644 --- a/source/common/logger/ur_logger.hpp +++ b/source/common/logger/ur_logger.hpp @@ -43,6 +43,11 @@ inline void error(const char *format, Args &&...args) { get_logger().log(logger::Level::ERR, format, std::forward(args)...); } +template +inline void always(const char *format, Args &&...args) { + get_logger().always(format, std::forward(args)...); +} + inline void setLevel(logger::Level level) { get_logger().setLevel(level); } inline void setFlushLevel(logger::Level level) { diff --git a/source/common/logger/ur_logger_details.hpp b/source/common/logger/ur_logger_details.hpp index 4759a2fd24..ac2fc3d80e 100644 --- a/source/common/logger/ur_logger_details.hpp +++ b/source/common/logger/ur_logger_details.hpp @@ -51,6 +51,13 @@ class Logger { log(logger::Level::ERR, format, std::forward(args)...); } + template + void always(const char *format, Args &&...args) { + if (sink) { + sink->log(level, format, std::forward(args)...); + } + } + template void log(logger::Level level, const char *format, Args &&...args) { if (level < this->level) { diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index 6640c46bc9..7d2102c8d2 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -228,16 +228,16 @@ void SanitizerInterceptor::postLaunchKernel(ur_kernel_handle_t Kernel, const char *File = AH->File[0] ? AH->File : ""; const char *Func = AH->Func[0] ? AH->Func : ""; - fprintf(stderr, "\n====ERROR: DeviceSanitizer: %s on %s\n\n", - DeviceSanitizerFormat(AH->ErrorType), - DeviceSanitizerFormat(AH->MemoryType)); - fprintf(stderr, - "%s of size %u at kernel <%s> LID(%lu, %lu, %lu) GID(%lu, " - "%lu, %lu)\n", - AH->IsWrite ? "WRITE" : "READ", AH->AccessSize, Func, AH->LID0, - AH->LID1, AH->LID2, AH->GID0, AH->GID1, AH->GID2); - fprintf(stderr, " #0 %s %s:%d\n", Func, File, AH->Line); - fflush(stderr); + context.logger.always("\n====ERROR: DeviceSanitizer: %s on %s\n\n", + DeviceSanitizerFormat(AH->ErrorType), + DeviceSanitizerFormat(AH->MemoryType)); + context.logger.always( + stderr, + "%s of size %u at kernel <%s> LID(%lu, %lu, %lu) GID(%lu, " + "%lu, %lu)\n", + AH->IsWrite ? "WRITE" : "READ", AH->AccessSize, Func, AH->LID0, + AH->LID1, AH->LID2, AH->GID0, AH->GID1, AH->GID2); + context.logger.always(stderr, " #0 %s %s:%d\n", Func, File, AH->Line); if (!AH->IsRecover) { abort(); } From a1c4ddce3b15625f8490516ec3ca5b9f25f0f553 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Tue, 12 Dec 2023 02:56:26 -0800 Subject: [PATCH 58/72] add some comments --- .../layers/sanitizer/asan_interceptor.cpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index 7d2102c8d2..9f486a9a83 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -84,6 +84,13 @@ ur_program_handle_t getProgram(ur_kernel_handle_t Kernel) { } // namespace +/// The memory chunk allocated from the underlying allocator looks like this: +/// L L L L L L U U U U U U R R +/// L -- left redzone words (0 or more bytes) +/// U -- user memory. +/// R -- right redzone (0 or more bytes) +/// +/// ref: "compiler-rt/lib/asan/asan_allocator.cpp" Allocator::Allocate ur_result_t SanitizerInterceptor::allocateMemory( ur_context_handle_t Context, ur_device_handle_t Device, const ur_usm_desc_t *Properties, ur_usm_pool_handle_t Pool, size_t Size, @@ -232,12 +239,11 @@ void SanitizerInterceptor::postLaunchKernel(ur_kernel_handle_t Kernel, DeviceSanitizerFormat(AH->ErrorType), DeviceSanitizerFormat(AH->MemoryType)); context.logger.always( - stderr, "%s of size %u at kernel <%s> LID(%lu, %lu, %lu) GID(%lu, " "%lu, %lu)\n", AH->IsWrite ? "WRITE" : "READ", AH->AccessSize, Func, AH->LID0, AH->LID1, AH->LID2, AH->GID0, AH->GID1, AH->GID2); - context.logger.always(stderr, " #0 %s %s:%d\n", Func, File, AH->Line); + context.logger.always(" #0 %s %s:%d\n", Func, File, AH->Line); if (!AH->IsRecover) { abort(); } @@ -377,6 +383,13 @@ ur_result_t SanitizerInterceptor::enqueueMemSetShadow( return UR_RESULT_SUCCESS; } +/// Each 8 bytes of application memory are mapped into one byte of shadow memory +/// The meaning of that byte: +/// - Negative: All bytes are not accessible (poisoned) +/// - 0: All bytes are accessible +/// - 1 <= k <= 7: Only the first k bytes is accessible +/// +/// ref: https://github.com/google/sanitizers/wiki/AddressSanitizerAlgorithm#mapping ur_result_t SanitizerInterceptor::enqueueAllocInfo( ur_context_handle_t Context, ur_device_handle_t Device, ur_queue_handle_t Queue, std::shared_ptr &AllocInfo, From 40b5ef61d0f574c096ddbe1dd665460283f5ab49 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Mon, 18 Dec 2023 19:22:22 -0800 Subject: [PATCH 59/72] fix logger --- source/common/logger/ur_logger_details.hpp | 2 +- source/loader/layers/sanitizer/asan_interceptor.cpp | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/source/common/logger/ur_logger_details.hpp b/source/common/logger/ur_logger_details.hpp index ac2fc3d80e..ea2143f2d7 100644 --- a/source/common/logger/ur_logger_details.hpp +++ b/source/common/logger/ur_logger_details.hpp @@ -54,7 +54,7 @@ class Logger { template void always(const char *format, Args &&...args) { if (sink) { - sink->log(level, format, std::forward(args)...); + sink->log(Level::QUIET, format, std::forward(args)...); } } diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index 9f486a9a83..597e8667e5 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -156,7 +156,8 @@ ur_result_t SanitizerInterceptor::allocateMemory( context.logger.info( "AllocInfos(AllocBegin={}, User={}-{}, NeededSize={}, Type={})", - AllocBegin, UserBegin, UserEnd, NeededSize, Type); + (void *)AllocBegin, (void *)UserBegin, (void *)UserEnd, NeededSize, + Type); return UR_RESULT_SUCCESS; } @@ -278,8 +279,13 @@ ur_result_t SanitizerInterceptor::allocShadowMemory( constexpr size_t SHADOW_SIZE = 1ULL << 46; // TODO: Protect Bad Zone - UR_CALL(context.urDdiTable.VirtualMem.pfnReserve( - Context, nullptr, SHADOW_SIZE, (void **)&DeviceInfo->ShadowOffset)); + auto Result = context.urDdiTable.VirtualMem.pfnReserve( + Context, nullptr, SHADOW_SIZE, (void **)&DeviceInfo->ShadowOffset); + if (Result != UR_RESULT_SUCCESS) { + context.logger.error( + "Shadow memory is allocated failed on PVC ({})", Result); + return Result; + } DeviceInfo->ShadowOffsetEnd = DeviceInfo->ShadowOffset + SHADOW_SIZE; } else { From 0b007981d5214fcf652802745b08bc1e1708fbf5 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Mon, 18 Dec 2023 19:24:13 -0800 Subject: [PATCH 60/72] improve logger --- source/loader/layers/sanitizer/asan_interceptor.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index 597e8667e5..a48bfa54f4 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -236,15 +236,15 @@ void SanitizerInterceptor::postLaunchKernel(ur_kernel_handle_t Kernel, const char *File = AH->File[0] ? AH->File : ""; const char *Func = AH->Func[0] ? AH->Func : ""; - context.logger.always("\n====ERROR: DeviceSanitizer: %s on %s\n\n", + context.logger.always("====ERROR: DeviceSanitizer: %s on %s", DeviceSanitizerFormat(AH->ErrorType), DeviceSanitizerFormat(AH->MemoryType)); context.logger.always( "%s of size %u at kernel <%s> LID(%lu, %lu, %lu) GID(%lu, " - "%lu, %lu)\n", + "%lu, %lu)", AH->IsWrite ? "WRITE" : "READ", AH->AccessSize, Func, AH->LID0, AH->LID1, AH->LID2, AH->GID0, AH->GID1, AH->GID2); - context.logger.always(" #0 %s %s:%d\n", Func, File, AH->Line); + context.logger.always(" #0 %s %s:%d", Func, File, AH->Line); if (!AH->IsRecover) { abort(); } From c42bd9adebe1c0bbba273e9d79270bc9daad8cd0 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Wed, 20 Dec 2023 19:11:14 -0800 Subject: [PATCH 61/72] don't print prefix when logger::Level::QUIET --- source/common/logger/ur_logger_details.hpp | 3 ++- source/common/logger/ur_sinks.hpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/common/logger/ur_logger_details.hpp b/source/common/logger/ur_logger_details.hpp index ea2143f2d7..6ff279ad1a 100644 --- a/source/common/logger/ur_logger_details.hpp +++ b/source/common/logger/ur_logger_details.hpp @@ -54,7 +54,8 @@ class Logger { template void always(const char *format, Args &&...args) { if (sink) { - sink->log(Level::QUIET, format, std::forward(args)...); + sink->log(logger::Level::QUIET, format, + std::forward(args)...); } } diff --git a/source/common/logger/ur_sinks.hpp b/source/common/logger/ur_sinks.hpp index cb8c751e4d..db30f3c8ed 100644 --- a/source/common/logger/ur_sinks.hpp +++ b/source/common/logger/ur_sinks.hpp @@ -22,7 +22,7 @@ class Sink { template void log(logger::Level level, const char *fmt, Args &&...args) { std::ostringstream buffer; - if (!skip_prefix) { + if (!skip_prefix && level != logger::Level::QUIET) { buffer << "<" << logger_name << ">" << "[" << level_to_str(level) << "]: "; } From bc8cdb221d392f59e3d0390c514c868eef292e09 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Wed, 20 Dec 2023 19:11:54 -0800 Subject: [PATCH 62/72] small fix --- .../layers/sanitizer/asan_interceptor.cpp | 23 +++++++++++-------- source/loader/layers/sanitizer/common.hpp | 4 ++-- .../sanitizer/device_sanitizer_report.hpp | 12 +++++----- source/loader/layers/sanitizer/ur_sanddi.cpp | 1 - 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index a48bfa54f4..b5dc1dbbe1 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -124,7 +124,8 @@ ur_result_t SanitizerInterceptor::allocateMemory( UR_CALL(context.urDdiTable.USM.pfnSharedAlloc( Context, Device, Properties, Pool, NeededSize, &Allocated)); } else { - die("SanitizerInterceptor: unsupport memory type"); + context.logger.error("Unsupport memory type"); + return UR_RESULT_ERROR_INVALID_ARGUMENT; } // Copy from LLVM compiler-rt/lib/asan @@ -236,15 +237,15 @@ void SanitizerInterceptor::postLaunchKernel(ur_kernel_handle_t Kernel, const char *File = AH->File[0] ? AH->File : ""; const char *Func = AH->Func[0] ? AH->Func : ""; - context.logger.always("====ERROR: DeviceSanitizer: %s on %s", + context.logger.always("\n====ERROR: DeviceSanitizer: {} on {}", DeviceSanitizerFormat(AH->ErrorType), DeviceSanitizerFormat(AH->MemoryType)); context.logger.always( - "%s of size %u at kernel <%s> LID(%lu, %lu, %lu) GID(%lu, " - "%lu, %lu)", + "{} of size {} at kernel <{}> LID({}, {}, {}) GID({}, " + "{}, {})", AH->IsWrite ? "WRITE" : "READ", AH->AccessSize, Func, AH->LID0, AH->LID1, AH->LID2, AH->GID0, AH->GID1, AH->GID2); - context.logger.always(" #0 %s %s:%d", Func, File, AH->Line); + context.logger.always(" #0 {} {}:{}", Func, File, AH->Line); if (!AH->IsRecover) { abort(); } @@ -270,7 +271,8 @@ std::string SanitizerInterceptor::getKernelName(ur_kernel_handle_t Kernel) { ur_result_t SanitizerInterceptor::allocShadowMemory( ur_context_handle_t Context, std::shared_ptr &DeviceInfo) { if (DeviceInfo->Type == DeviceType::CPU) { - die("Unsupport device type"); + context.logger.error("Unsupport device type"); + return UR_RESULT_ERROR_INVALID_ARGUMENT; } else if (DeviceInfo->Type == DeviceType::GPU_PVC) { /// SHADOW MEMORY MAPPING (PVC, with CPU 47bit) /// Host/Shared USM : 0x0 ~ 0x0fff_ffff_ffff @@ -289,7 +291,8 @@ ur_result_t SanitizerInterceptor::allocShadowMemory( DeviceInfo->ShadowOffsetEnd = DeviceInfo->ShadowOffset + SHADOW_SIZE; } else { - die("Unsupport device type"); + context.logger.error("Unsupport device type"); + return UR_RESULT_ERROR_INVALID_ARGUMENT; } context.logger.info("Device ShadowOffset: {} - {}", (void *)DeviceInfo->ShadowOffset, @@ -309,7 +312,8 @@ ur_result_t SanitizerInterceptor::enqueueMemSetShadow( auto DeviceInfo = ContextInfo->getDeviceInfo(Device); if (DeviceInfo->Type == DeviceType::CPU) { - die("Unsupport device type"); + context.logger.error("Unsupport device type"); + return UR_RESULT_ERROR_INVALID_ARGUMENT; } else if (DeviceInfo->Type == DeviceType::GPU_PVC) { ur_event_handle_t InternalEvent{}; ur_event_handle_t *Event = OutEvent ? OutEvent : &InternalEvent; @@ -384,7 +388,8 @@ ur_result_t SanitizerInterceptor::enqueueMemSetShadow( return URes; } } else { - die("Unsupport device type"); + context.logger.error("Unsupport device type"); + return UR_RESULT_ERROR_INVALID_ARGUMENT; } return UR_RESULT_SUCCESS; } diff --git a/source/loader/layers/sanitizer/common.hpp b/source/loader/layers/sanitizer/common.hpp index 05ac3f0c07..6acd661de0 100644 --- a/source/loader/layers/sanitizer/common.hpp +++ b/source/loader/layers/sanitizer/common.hpp @@ -51,7 +51,7 @@ inline constexpr bool IsAligned(uptr a, uptr alignment) { // Valid redzone sizes are 16, 32, 64, ... 2048, so we encode them in 3 bits. // We use adaptive redzones: for larger allocation larger redzones are used. inline constexpr u32 RZLog2Size(u32 rz_log) { - // CHECK_LT(rz_log, 8); + assert(rz_log < 8); return 16 << rz_log; } @@ -69,7 +69,7 @@ inline constexpr uptr ComputeRZLog(uptr user_requested_size) { // ================================================================ -// Trace an internal PI call; returns in case of an error. +// Trace an internal UR call; returns in case of an error. #define UR_CALL(Call) \ { \ if (PrintTrace) \ diff --git a/source/loader/layers/sanitizer/device_sanitizer_report.hpp b/source/loader/layers/sanitizer/device_sanitizer_report.hpp index abe2949db4..6bed68aa04 100644 --- a/source/loader/layers/sanitizer/device_sanitizer_report.hpp +++ b/source/loader/layers/sanitizer/device_sanitizer_report.hpp @@ -18,10 +18,10 @@ namespace ur_sanitizer_layer { enum class DeviceSanitizerErrorType : int32_t { UNKNOWN, - OUT_OF_BOUND, + OUT_OF_BOUNDS, MISALIGNED, USE_AFTER_FREE, - OUT_OF_SHADOW_BOUND, + OUT_OF_SHADOW_BOUNDS, }; enum class DeviceSanitizerMemoryType : int32_t { @@ -79,14 +79,14 @@ const char *DeviceSanitizerFormat(DeviceSanitizerMemoryType MemoryType) { const char *DeviceSanitizerFormat(DeviceSanitizerErrorType ErrorType) { switch (ErrorType) { - case DeviceSanitizerErrorType::OUT_OF_BOUND: - return "out-of-bound-access"; + case DeviceSanitizerErrorType::OUT_OF_BOUNDS: + return "out-of-bounds-access"; case DeviceSanitizerErrorType::MISALIGNED: return "misaligned-access"; case DeviceSanitizerErrorType::USE_AFTER_FREE: return "use-after-free"; - case DeviceSanitizerErrorType::OUT_OF_SHADOW_BOUND: - return "out-of-shadow-bound-access"; + case DeviceSanitizerErrorType::OUT_OF_SHADOW_BOUNDS: + return "out-of-shadow-bounds-access"; default: return "unknown-error"; } diff --git a/source/loader/layers/sanitizer/ur_sanddi.cpp b/source/loader/layers/sanitizer/ur_sanddi.cpp index 3c2be0cd7f..e1fdc3b5c2 100644 --- a/source/loader/layers/sanitizer/ur_sanddi.cpp +++ b/source/loader/layers/sanitizer/ur_sanddi.cpp @@ -453,7 +453,6 @@ ur_result_t context_t::init(ur_dditable_t *dditable, } } - // FIXME: Interceptor needs use some of APIs that aren't registered, maybe just copy needed APIs? urDdiTable = *dditable; if (UR_RESULT_SUCCESS == result) { From 3bede09beac9ecbb0ff1e7fd8b67de3eddaafeac Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Wed, 20 Dec 2023 19:15:04 -0800 Subject: [PATCH 63/72] small fix --- source/loader/layers/sanitizer/asan_interceptor.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index b5dc1dbbe1..b0535e2a7b 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -340,12 +340,12 @@ ur_result_t SanitizerInterceptor::enqueueMemSetShadow( auto URes = context.urDdiTable.PhysicalMem.pfnCreate( Context, Device, PageSize, &Desc, &PhysicalMem); if (URes != UR_RESULT_SUCCESS) { - context.logger.error("zePhysicalMemCreate(): {}", URes); + context.logger.error("urPhysicalMemCreate(): {}", URes); return URes; } } - context.logger.debug("zeVirtualMemMap: {} ~ {}", + context.logger.debug("urVirtualMemMap: {} ~ {}", (void *)MappedPtr, (void *)(MappedPtr + PageSize - 1)); @@ -354,7 +354,7 @@ ur_result_t SanitizerInterceptor::enqueueMemSetShadow( Context, (void *)MappedPtr, PageSize, PhysicalMem, 0, UR_VIRTUAL_MEM_ACCESS_FLAG_READ_WRITE); if (URes != UR_RESULT_SUCCESS) { - context.logger.debug("zeVirtualMemMap(): {}", URes); + context.logger.debug("urVirtualMemMap(): {}", URes); } // Initialize to zero From 5dc2d5ce16ad82a4e383bbb4bc05658f7cb55922 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Wed, 20 Dec 2023 19:21:41 -0800 Subject: [PATCH 64/72] fix KernelNameBuf --- source/loader/layers/sanitizer/asan_interceptor.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index b0535e2a7b..5a822b0c1a 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -258,14 +258,13 @@ std::string SanitizerInterceptor::getKernelName(ur_kernel_handle_t Kernel) { Kernel, UR_KERNEL_INFO_FUNCTION_NAME, 0, nullptr, &KernelNameSize); assert(Res == UR_RESULT_SUCCESS); - std::vector KernelNameBuf(KernelNameSize + 1); + std::vector KernelNameBuf(KernelNameSize); Res = context.urDdiTable.Kernel.pfnGetInfo( Kernel, UR_KERNEL_INFO_FUNCTION_NAME, KernelNameSize, KernelNameBuf.data(), nullptr); assert(Res == UR_RESULT_SUCCESS); - KernelNameBuf[KernelNameSize] = '\0'; - return std::string(KernelNameBuf.data(), KernelNameSize); + return std::string(KernelNameBuf.data(), KernelNameSize - 1); } ur_result_t SanitizerInterceptor::allocShadowMemory( From 30c0bc13369f653e9cbbb94c775592b18c4c61de Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Mon, 1 Jan 2024 21:58:33 -0800 Subject: [PATCH 65/72] follow the review comments --- .../layers/sanitizer/asan_interceptor.cpp | 36 +++++++++---------- .../layers/sanitizer/asan_interceptor.hpp | 3 +- source/loader/layers/sanitizer/common.hpp | 10 +++--- .../sanitizer/device_sanitizer_report.hpp | 4 +-- 4 files changed, 24 insertions(+), 29 deletions(-) diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index 5a822b0c1a..ca73ea10a9 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -7,7 +7,7 @@ * See LICENSE.TXT * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception * - * @file ur_sanitizer_layer.cpp + * @file asan_interceptor.cpp * */ @@ -34,10 +34,6 @@ const auto kSPIR_DeviceSanitizerReportMem = "__DeviceSanitizerReportMem"; DeviceSanitizerReport SPIR_DeviceSanitizerReportMem; -// uptr MemToShadow_CPU(uptr USM_SHADOW_BASE, uptr UPtr) { -// return USM_SHADOW_BASE + (UPtr >> 3); -// } - uptr MemToShadow_PVC(uptr USM_SHADOW_BASE, uptr UPtr) { if (UPtr & 0xFF00000000000000ULL) { // Device USM return USM_SHADOW_BASE + 0x200000000000ULL + @@ -47,14 +43,6 @@ uptr MemToShadow_PVC(uptr USM_SHADOW_BASE, uptr UPtr) { } } -// uptr MemToShadow_DG2(uptr USM_SHADOW_BASE, uptr UPtr) { -// if (UPtr & (~0xFFFFFFFFFFFFULL)) { // Device USM -// return USM_SHADOW_BASE + ((UPtr & 0xFFFFFFFFFFFFULL) >> 3); -// } else { -// return USM_SHADOW_BASE + (UPtr >> 3); -// } -// } - ur_context_handle_t getContext(ur_queue_handle_t Queue) { ur_context_handle_t Context; [[maybe_unused]] auto Result = context.urDdiTable.Queue.pfnGetInfo( @@ -196,7 +184,9 @@ ur_result_t SanitizerInterceptor::releaseMemory(ur_context_handle_t Context, bool SanitizerInterceptor::preLaunchKernel(ur_kernel_handle_t Kernel, ur_queue_handle_t Queue, - ur_event_handle_t &Event) { + ur_event_handle_t *Event) { + assert(Event != nullptr); + prepareLaunch(Queue, Kernel); UR_CALL(updateShadowMemory(Queue)); @@ -207,7 +197,7 @@ bool SanitizerInterceptor::preLaunchKernel(ur_kernel_handle_t Kernel, auto QueueInfo = ContextInfo->getQueueInfo(Queue); std::scoped_lock Guard(QueueInfo->Mutex); - Event = QueueInfo->LastEvent; + *Event = QueueInfo->LastEvent; QueueInfo->LastEvent = nullptr; return true; @@ -224,10 +214,15 @@ void SanitizerInterceptor::postLaunchKernel(ur_kernel_handle_t Kernel, auto Result = context.urDdiTable.Enqueue.pfnDeviceGlobalVariableRead( Queue, Program, kSPIR_DeviceSanitizerReportMem, true, sizeof(SPIR_DeviceSanitizerReportMem), 0, - &SPIR_DeviceSanitizerReportMem, 1, Event, &ReadEvent); + &SPIR_DeviceSanitizerReportMem, Event ? 1 : 0, Event, &ReadEvent); if (Result == UR_RESULT_SUCCESS) { - *Event = ReadEvent; + if (Event) { + *Event = ReadEvent; + } else { + [[maybe_unused]] auto Result = context.urDdiTable.Event.pfnWait(1, &ReadEvent); + assert(Result == UR_RESULT_SUCCESS); + } auto AH = &SPIR_DeviceSanitizerReportMem; if (!AH->Flag) { @@ -284,7 +279,7 @@ ur_result_t SanitizerInterceptor::allocShadowMemory( Context, nullptr, SHADOW_SIZE, (void **)&DeviceInfo->ShadowOffset); if (Result != UR_RESULT_SUCCESS) { context.logger.error( - "Shadow memory is allocated failed on PVC ({})", Result); + "Failed to allocate shadow memory on PVC: {}", Result); return Result; } @@ -293,7 +288,7 @@ ur_result_t SanitizerInterceptor::allocShadowMemory( context.logger.error("Unsupport device type"); return UR_RESULT_ERROR_INVALID_ARGUMENT; } - context.logger.info("Device ShadowOffset: {} - {}", + context.logger.info("ShadowMemory(Global): {} - {}", (void *)DeviceInfo->ShadowOffset, (void *)DeviceInfo->ShadowOffsetEnd); return UR_RESULT_SUCCESS; @@ -361,7 +356,7 @@ ur_result_t SanitizerInterceptor::enqueueMemSetShadow( // Reset PhysicalMem to null since it's been mapped PhysicalMem = nullptr; - const char Pattern[] = {kUnkownRedzoneMagic}; + const char Pattern[] = {0}; auto URes = context.urDdiTable.Enqueue.pfnUSMFill( Queue, (void *)MappedPtr, 1, Pattern, PageSize, @@ -504,6 +499,7 @@ ur_result_t SanitizerInterceptor::addContext(ur_context_handle_t Context) { ContextInfo->DeviceMap.emplace(nullptr, std::move(DeviceInfo)); std::scoped_lock Guard(m_ContextMapMutex); + assert(m_ContextMap.find(Context) == m_ContextMap.end()); m_ContextMap.emplace(Context, std::move(ContextInfo)); return UR_RESULT_SUCCESS; diff --git a/source/loader/layers/sanitizer/asan_interceptor.hpp b/source/loader/layers/sanitizer/asan_interceptor.hpp index d2afc7d0c8..dd4ddc6add 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.hpp +++ b/source/loader/layers/sanitizer/asan_interceptor.hpp @@ -83,7 +83,6 @@ struct ContextInfo { class SanitizerInterceptor { public: - SanitizerInterceptor() {} ur_result_t allocateMemory(ur_context_handle_t Context, ur_device_handle_t Device, @@ -93,7 +92,7 @@ class SanitizerInterceptor { ur_result_t releaseMemory(ur_context_handle_t Context, void *Ptr); bool preLaunchKernel(ur_kernel_handle_t Kernel, ur_queue_handle_t Queue, - ur_event_handle_t &Event); + ur_event_handle_t *Event); void postLaunchKernel(ur_kernel_handle_t Kernel, ur_queue_handle_t Queue, ur_event_handle_t *Event); diff --git a/source/loader/layers/sanitizer/common.hpp b/source/loader/layers/sanitizer/common.hpp index 6acd661de0..e180f00a14 100644 --- a/source/loader/layers/sanitizer/common.hpp +++ b/source/loader/layers/sanitizer/common.hpp @@ -23,12 +23,12 @@ namespace ur_sanitizer_layer { // ================================================================ // Copy from LLVM compiler-rt/lib/asan -typedef uintptr_t uptr; -typedef unsigned char u8; -typedef unsigned int u32; +using uptr = uintptr_t; +using u8 = unsigned char; +using u32 = unsigned int; -#define ASAN_SHADOW_SCALE 3 -#define ASAN_SHADOW_GRANULARITY (1ULL << ASAN_SHADOW_SCALE) +constexpr unsigned ASAN_SHADOW_SCALE = 3; +constexpr unsigned ASAN_SHADOW_GRANULARITY = 1ULL << ASAN_SHADOW_SCALE; inline constexpr bool IsPowerOfTwo(uptr x) { return (x & (x - 1)) == 0 && x != 0; diff --git a/source/loader/layers/sanitizer/device_sanitizer_report.hpp b/source/loader/layers/sanitizer/device_sanitizer_report.hpp index 6bed68aa04..11ae721434 100644 --- a/source/loader/layers/sanitizer/device_sanitizer_report.hpp +++ b/source/loader/layers/sanitizer/device_sanitizer_report.hpp @@ -58,7 +58,7 @@ struct DeviceSanitizerReport { bool IsRecover = false; }; -const char *DeviceSanitizerFormat(DeviceSanitizerMemoryType MemoryType) { +inline const char *DeviceSanitizerFormat(DeviceSanitizerMemoryType MemoryType) { switch (MemoryType) { case DeviceSanitizerMemoryType::USM_DEVICE: return "USM Device Memory"; @@ -77,7 +77,7 @@ const char *DeviceSanitizerFormat(DeviceSanitizerMemoryType MemoryType) { } } -const char *DeviceSanitizerFormat(DeviceSanitizerErrorType ErrorType) { +inline const char *DeviceSanitizerFormat(DeviceSanitizerErrorType ErrorType) { switch (ErrorType) { case DeviceSanitizerErrorType::OUT_OF_BOUNDS: return "out-of-bounds-access"; From 86d405614f955dbfe13bfb6d2dbbd68b3e70d71d Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Mon, 1 Jan 2024 21:59:59 -0800 Subject: [PATCH 66/72] fix build --- source/loader/layers/sanitizer/ur_sanddi.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/loader/layers/sanitizer/ur_sanddi.cpp b/source/loader/layers/sanitizer/ur_sanddi.cpp index e1fdc3b5c2..4da12e5426 100644 --- a/source/loader/layers/sanitizer/ur_sanddi.cpp +++ b/source/loader/layers/sanitizer/ur_sanddi.cpp @@ -188,7 +188,7 @@ __urdlllocal ur_result_t UR_APICALL urEnqueueKernelLaunch( } // launchKernel must append to num_events_in_wait_list, not prepend - context.interceptor->preLaunchKernel(hKernel, hQueue, lk_event); + context.interceptor->preLaunchKernel(hKernel, hQueue, &lk_event); if (lk_event) { events.push_back(lk_event); } From 49e2f4fcd3ec08c342a764f13670b98a6564bc31 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Mon, 1 Jan 2024 22:33:48 -0800 Subject: [PATCH 67/72] follow the review comments --- .../layers/sanitizer/asan_interceptor.cpp | 28 +++++++++---------- .../layers/sanitizer/asan_interceptor.hpp | 10 +++---- source/loader/layers/sanitizer/ur_sanddi.cpp | 14 +++++----- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index ca73ea10a9..4e4f09b580 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -21,16 +21,16 @@ namespace { // These magic values are written to shadow for better error // reporting. -const int kUsmDeviceRedzoneMagic = (char)0x81; -const int kUsmHostRedzoneMagic = (char)0x82; -const int kUsmSharedRedzoneMagic = (char)0x83; -const int kMemBufferRedzoneMagic = (char)0x84; -const int kUnkownRedzoneMagic = (char)0x8F; +constexpr int kUsmDeviceRedzoneMagic = (char)0x81; +constexpr int kUsmHostRedzoneMagic = (char)0x82; +constexpr int kUsmSharedRedzoneMagic = (char)0x83; +constexpr int kMemBufferRedzoneMagic = (char)0x84; +constexpr int kUnkownRedzoneMagic = (char)0x8F; -const auto kSPIR_AsanShadowMemoryGlobalStart = "__AsanShadowMemoryGlobalStart"; -const auto kSPIR_AsanShadowMemoryGlobalEnd = "__AsanShadowMemoryGlobalEnd"; +constexpr auto kSPIR_AsanShadowMemoryGlobalStart = "__AsanShadowMemoryGlobalStart"; +constexpr auto kSPIR_AsanShadowMemoryGlobalEnd = "__AsanShadowMemoryGlobalEnd"; -const auto kSPIR_DeviceSanitizerReportMem = "__DeviceSanitizerReportMem"; +constexpr auto kSPIR_DeviceSanitizerReportMem = "__DeviceSanitizerReportMem"; DeviceSanitizerReport SPIR_DeviceSanitizerReportMem; @@ -157,7 +157,7 @@ ur_result_t SanitizerInterceptor::releaseMemory(ur_context_handle_t Context, std::shared_lock Guard(ContextInfo->Mutex); - auto Addr = (uptr)Ptr; + auto Addr = reinterpret_cast(Ptr); // Find the last element is not greater than key auto AllocInfoIt = ContextInfo->AllocatedUSMMap.upper_bound((uptr)Addr); if (AllocInfoIt == ContextInfo->AllocatedUSMMap.begin()) { @@ -484,7 +484,7 @@ ur_result_t SanitizerInterceptor::updateShadowMemory(ur_queue_handle_t Queue) { return UR_RESULT_SUCCESS; } -ur_result_t SanitizerInterceptor::addContext(ur_context_handle_t Context) { +ur_result_t SanitizerInterceptor::insertContext(ur_context_handle_t Context) { auto ContextInfo = std::make_shared(); // Host Device @@ -505,14 +505,14 @@ ur_result_t SanitizerInterceptor::addContext(ur_context_handle_t Context) { return UR_RESULT_SUCCESS; } -ur_result_t SanitizerInterceptor::removeContext(ur_context_handle_t Context) { +ur_result_t SanitizerInterceptor::eraseContext(ur_context_handle_t Context) { std::scoped_lock Guard(m_ContextMapMutex); assert(m_ContextMap.find(Context) != m_ContextMap.end()); m_ContextMap.erase(Context); return UR_RESULT_SUCCESS; } -ur_result_t SanitizerInterceptor::addDevice(ur_context_handle_t Context, +ur_result_t SanitizerInterceptor::insertDevice(ur_context_handle_t Context, ur_device_handle_t Device) { auto DeviceInfo = std::make_shared(); @@ -546,7 +546,7 @@ ur_result_t SanitizerInterceptor::addDevice(ur_context_handle_t Context, return UR_RESULT_SUCCESS; } -ur_result_t SanitizerInterceptor::addQueue(ur_context_handle_t Context, +ur_result_t SanitizerInterceptor::insertQueue(ur_context_handle_t Context, ur_queue_handle_t Queue) { auto QueueInfo = std::make_shared(); QueueInfo->LastEvent = nullptr; @@ -558,7 +558,7 @@ ur_result_t SanitizerInterceptor::addQueue(ur_context_handle_t Context, return UR_RESULT_SUCCESS; } -ur_result_t SanitizerInterceptor::removeQueue(ur_context_handle_t Context, +ur_result_t SanitizerInterceptor::eraseQueue(ur_context_handle_t Context, ur_queue_handle_t Queue) { auto ContextInfo = getContextInfo(Context); std::scoped_lock Guard(ContextInfo->Mutex); diff --git a/source/loader/layers/sanitizer/asan_interceptor.hpp b/source/loader/layers/sanitizer/asan_interceptor.hpp index dd4ddc6add..b4bca80a0b 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.hpp +++ b/source/loader/layers/sanitizer/asan_interceptor.hpp @@ -96,14 +96,14 @@ class SanitizerInterceptor { void postLaunchKernel(ur_kernel_handle_t Kernel, ur_queue_handle_t Queue, ur_event_handle_t *Event); - ur_result_t addContext(ur_context_handle_t Context); - ur_result_t removeContext(ur_context_handle_t Context); + ur_result_t insertContext(ur_context_handle_t Context); + ur_result_t eraseContext(ur_context_handle_t Context); - ur_result_t addDevice(ur_context_handle_t Context, + ur_result_t insertDevice(ur_context_handle_t Context, ur_device_handle_t Device); - ur_result_t addQueue(ur_context_handle_t Context, ur_queue_handle_t Queue); - ur_result_t removeQueue(ur_context_handle_t Context, + ur_result_t insertQueue(ur_context_handle_t Context, ur_queue_handle_t Queue); + ur_result_t eraseQueue(ur_context_handle_t Context, ur_queue_handle_t Queue); private: diff --git a/source/loader/layers/sanitizer/ur_sanddi.cpp b/source/loader/layers/sanitizer/ur_sanddi.cpp index 4da12e5426..d5b3360cd2 100644 --- a/source/loader/layers/sanitizer/ur_sanddi.cpp +++ b/source/loader/layers/sanitizer/ur_sanddi.cpp @@ -116,7 +116,7 @@ __urdlllocal ur_result_t UR_APICALL urQueueCreate( ur_result_t result = pfnCreate(hContext, hDevice, pProperties, phQueue); if (result == UR_RESULT_SUCCESS) { - result = context.interceptor->addQueue(hContext, *phQueue); + result = context.interceptor->insertQueue(hContext, *phQueue); } return result; @@ -137,7 +137,7 @@ __urdlllocal ur_result_t UR_APICALL urQueueRelease( UR_CALL(context.urDdiTable.Queue.pfnGetInfo(hQueue, UR_QUEUE_INFO_CONTEXT, sizeof(ur_context_handle_t), &hContext, nullptr)); - UR_CALL(context.interceptor->removeQueue(hContext, hQueue)); + UR_CALL(context.interceptor->eraseQueue(hContext, hQueue)); ur_result_t result = pfnRelease(hQueue); @@ -226,12 +226,12 @@ __urdlllocal ur_result_t UR_APICALL urContextCreate( if (result == UR_RESULT_SUCCESS) { auto Context = *phContext; - result = context.interceptor->addContext(Context); + result = context.interceptor->insertContext(Context); if (result != UR_RESULT_SUCCESS) { return result; } for (uint32_t i = 0; i < numDevices; ++i) { - result = context.interceptor->addDevice(Context, phDevices[i]); + result = context.interceptor->insertDevice(Context, phDevices[i]); if (result != UR_RESULT_SUCCESS) { return result; } @@ -266,12 +266,12 @@ __urdlllocal ur_result_t UR_APICALL urContextCreateWithNativeHandle( if (result == UR_RESULT_SUCCESS) { auto Context = *phContext; - result = context.interceptor->addContext(Context); + result = context.interceptor->insertContext(Context); if (result != UR_RESULT_SUCCESS) { return result; } for (uint32_t i = 0; i < numDevices; ++i) { - result = context.interceptor->addDevice(Context, phDevices[i]); + result = context.interceptor->insertDevice(Context, phDevices[i]); if (result != UR_RESULT_SUCCESS) { return result; } @@ -292,7 +292,7 @@ __urdlllocal ur_result_t UR_APICALL urContextRelease( return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } - UR_CALL(context.interceptor->removeContext(hContext)); + UR_CALL(context.interceptor->eraseContext(hContext)); ur_result_t result = pfnRelease(hContext); return result; From 79998b965232ff40e3a0b3829dd57f5466eb23b3 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Tue, 2 Jan 2024 00:59:35 -0800 Subject: [PATCH 68/72] follow the review comments --- .../layers/sanitizer/asan_interceptor.cpp | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index 4e4f09b580..b0ede28e22 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -27,7 +27,8 @@ constexpr int kUsmSharedRedzoneMagic = (char)0x83; constexpr int kMemBufferRedzoneMagic = (char)0x84; constexpr int kUnkownRedzoneMagic = (char)0x8F; -constexpr auto kSPIR_AsanShadowMemoryGlobalStart = "__AsanShadowMemoryGlobalStart"; +constexpr auto kSPIR_AsanShadowMemoryGlobalStart = + "__AsanShadowMemoryGlobalStart"; constexpr auto kSPIR_AsanShadowMemoryGlobalEnd = "__AsanShadowMemoryGlobalEnd"; constexpr auto kSPIR_DeviceSanitizerReportMem = "__DeviceSanitizerReportMem"; @@ -220,7 +221,8 @@ void SanitizerInterceptor::postLaunchKernel(ur_kernel_handle_t Kernel, if (Event) { *Event = ReadEvent; } else { - [[maybe_unused]] auto Result = context.urDdiTable.Event.pfnWait(1, &ReadEvent); + [[maybe_unused]] auto Result = + context.urDdiTable.Event.pfnWait(1, &ReadEvent); assert(Result == UR_RESULT_SUCCESS); } @@ -278,8 +280,8 @@ ur_result_t SanitizerInterceptor::allocShadowMemory( auto Result = context.urDdiTable.VirtualMem.pfnReserve( Context, nullptr, SHADOW_SIZE, (void **)&DeviceInfo->ShadowOffset); if (Result != UR_RESULT_SUCCESS) { - context.logger.error( - "Failed to allocate shadow memory on PVC: {}", Result); + context.logger.error("Failed to allocate shadow memory on PVC: {}", + Result); return Result; } @@ -316,12 +318,18 @@ ur_result_t SanitizerInterceptor::enqueueMemSetShadow( uptr ShadowEnd = MemToShadow_PVC(DeviceInfo->ShadowOffset, Ptr + Size - 1); - // Maybe in future, we needn't to map physical memory manually - const bool IsNeedMapPhysicalMem = true; - - if (IsNeedMapPhysicalMem) { - // We use fixed GPU PageSize: 64KB - const size_t PageSize = 64 * 1024u; + { + static const size_t PageSize = [Context, Device]() { + size_t Size; + auto Result = + context.urDdiTable.VirtualMem.pfnGranularityGetInfo( + Context, Device, + UR_VIRTUAL_MEM_GRANULARITY_INFO_RECOMMENDED, + sizeof(Size), &Size, nullptr); + assert(Result == UR_RESULT_SUCCESS); + context.logger.info("PVC PageSize: {}", Size); + return Size; + }(); ur_physical_mem_properties_t Desc{ UR_STRUCTURE_TYPE_PHYSICAL_MEM_PROPERTIES, nullptr, 0}; @@ -513,7 +521,7 @@ ur_result_t SanitizerInterceptor::eraseContext(ur_context_handle_t Context) { } ur_result_t SanitizerInterceptor::insertDevice(ur_context_handle_t Context, - ur_device_handle_t Device) { + ur_device_handle_t Device) { auto DeviceInfo = std::make_shared(); // Query device type @@ -547,7 +555,7 @@ ur_result_t SanitizerInterceptor::insertDevice(ur_context_handle_t Context, } ur_result_t SanitizerInterceptor::insertQueue(ur_context_handle_t Context, - ur_queue_handle_t Queue) { + ur_queue_handle_t Queue) { auto QueueInfo = std::make_shared(); QueueInfo->LastEvent = nullptr; @@ -559,7 +567,7 @@ ur_result_t SanitizerInterceptor::insertQueue(ur_context_handle_t Context, } ur_result_t SanitizerInterceptor::eraseQueue(ur_context_handle_t Context, - ur_queue_handle_t Queue) { + ur_queue_handle_t Queue) { auto ContextInfo = getContextInfo(Context); std::scoped_lock Guard(ContextInfo->Mutex); assert(ContextInfo->QueueMap.find(Queue) != ContextInfo->QueueMap.end()); From 6cc4913ef7e7d443e2ee6053a24859abcf6198f5 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Wed, 3 Jan 2024 23:02:12 -0800 Subject: [PATCH 69/72] fix urEnqueueKernelLaunch & remove host deviceinfo --- .../layers/sanitizer/asan_interceptor.cpp | 52 +++++++------------ .../layers/sanitizer/asan_interceptor.hpp | 5 +- source/loader/layers/sanitizer/ur_sanddi.cpp | 17 +++--- 3 files changed, 31 insertions(+), 43 deletions(-) diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index b0ede28e22..83051caea0 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -88,11 +88,14 @@ ur_result_t SanitizerInterceptor::allocateMemory( assert(Alignment == 0 || IsPowerOfTwo(Alignment)); auto ContextInfo = getContextInfo(Context); - // Device is nullptr if Type == USMMemoryType::HOST - auto DeviceInfo = ContextInfo->getDeviceInfo(Device); + std::shared_ptr DeviceInfo; + if (Device) { + DeviceInfo = ContextInfo->getDeviceInfo(Device); + } if (Alignment == 0) { - Alignment = DeviceInfo->Alignment; + Alignment = + DeviceInfo ? DeviceInfo->Alignment : ASAN_SHADOW_GRANULARITY; } // Copy from LLVM compiler-rt/lib/asan @@ -133,9 +136,15 @@ ur_result_t SanitizerInterceptor::allocateMemory( USMAllocInfo{AllocBegin, UserBegin, UserEnd, NeededSize, Type}); // For updating shadow memory - { + if (DeviceInfo) { // device/shared USM std::scoped_lock Guard(DeviceInfo->Mutex); DeviceInfo->AllocInfos.emplace_back(AllocInfo); + } else { // host USM's AllocInfo needs to insert into all devices + for (auto &pair : ContextInfo->DeviceMap) { + auto DeviceInfo = pair.second; + std::scoped_lock Guard(DeviceInfo->Mutex); + DeviceInfo->AllocInfos.emplace_back(AllocInfo); + } } // For memory release @@ -185,9 +194,7 @@ ur_result_t SanitizerInterceptor::releaseMemory(ur_context_handle_t Context, bool SanitizerInterceptor::preLaunchKernel(ur_kernel_handle_t Kernel, ur_queue_handle_t Queue, - ur_event_handle_t *Event) { - assert(Event != nullptr); - + ur_event_handle_t &Event) { prepareLaunch(Queue, Kernel); UR_CALL(updateShadowMemory(Queue)); @@ -198,7 +205,7 @@ bool SanitizerInterceptor::preLaunchKernel(ur_kernel_handle_t Kernel, auto QueueInfo = ContextInfo->getQueueInfo(Queue); std::scoped_lock Guard(QueueInfo->Mutex); - *Event = QueueInfo->LastEvent; + Event = QueueInfo->LastEvent; QueueInfo->LastEvent = nullptr; return true; @@ -206,7 +213,7 @@ bool SanitizerInterceptor::preLaunchKernel(ur_kernel_handle_t Kernel, void SanitizerInterceptor::postLaunchKernel(ur_kernel_handle_t Kernel, ur_queue_handle_t Queue, - ur_event_handle_t *Event) { + ur_event_handle_t &Event) { auto Program = getProgram(Kernel); ur_event_handle_t ReadEvent{}; @@ -215,16 +222,10 @@ void SanitizerInterceptor::postLaunchKernel(ur_kernel_handle_t Kernel, auto Result = context.urDdiTable.Enqueue.pfnDeviceGlobalVariableRead( Queue, Program, kSPIR_DeviceSanitizerReportMem, true, sizeof(SPIR_DeviceSanitizerReportMem), 0, - &SPIR_DeviceSanitizerReportMem, Event ? 1 : 0, Event, &ReadEvent); + &SPIR_DeviceSanitizerReportMem, 1, &Event, &ReadEvent); if (Result == UR_RESULT_SUCCESS) { - if (Event) { - *Event = ReadEvent; - } else { - [[maybe_unused]] auto Result = - context.urDdiTable.Event.pfnWait(1, &ReadEvent); - assert(Result == UR_RESULT_SUCCESS); - } + Event = ReadEvent; auto AH = &SPIR_DeviceSanitizerReportMem; if (!AH->Flag) { @@ -476,12 +477,6 @@ ur_result_t SanitizerInterceptor::updateShadowMemory(ur_queue_handle_t Queue) { ur_event_handle_t LastEvent = QueueInfo->LastEvent; - // FIXME: Always update host USM, but it'd be better to update host USM - // selectively, or each devices once - for (auto &AllocInfo : HostInfo->AllocInfos) { - UR_CALL(enqueueAllocInfo(Context, Device, Queue, AllocInfo, LastEvent)); - } - for (auto &AllocInfo : DeviceInfo->AllocInfos) { UR_CALL(enqueueAllocInfo(Context, Device, Queue, AllocInfo, LastEvent)); } @@ -495,17 +490,6 @@ ur_result_t SanitizerInterceptor::updateShadowMemory(ur_queue_handle_t Queue) { ur_result_t SanitizerInterceptor::insertContext(ur_context_handle_t Context) { auto ContextInfo = std::make_shared(); - // Host Device - auto DeviceInfo = std::make_shared(); - DeviceInfo->Type = DeviceType::CPU; - DeviceInfo->Alignment = ASAN_SHADOW_GRANULARITY; - - // TODO: Check if host asan is enabled - DeviceInfo->ShadowOffset = 0; - DeviceInfo->ShadowOffsetEnd = 0; - - ContextInfo->DeviceMap.emplace(nullptr, std::move(DeviceInfo)); - std::scoped_lock Guard(m_ContextMapMutex); assert(m_ContextMap.find(Context) == m_ContextMap.end()); m_ContextMap.emplace(Context, std::move(ContextInfo)); diff --git a/source/loader/layers/sanitizer/asan_interceptor.hpp b/source/loader/layers/sanitizer/asan_interceptor.hpp index b4bca80a0b..a6838905d7 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.hpp +++ b/source/loader/layers/sanitizer/asan_interceptor.hpp @@ -70,7 +70,6 @@ struct ContextInfo { } ur_shared_mutex Mutex; - // Note: nullptr is host device std::unordered_map> DeviceMap; std::unordered_map> QueueMap; @@ -92,9 +91,9 @@ class SanitizerInterceptor { ur_result_t releaseMemory(ur_context_handle_t Context, void *Ptr); bool preLaunchKernel(ur_kernel_handle_t Kernel, ur_queue_handle_t Queue, - ur_event_handle_t *Event); + ur_event_handle_t &Event); void postLaunchKernel(ur_kernel_handle_t Kernel, ur_queue_handle_t Queue, - ur_event_handle_t *Event); + ur_event_handle_t &Event); ur_result_t insertContext(ur_context_handle_t Context); ur_result_t eraseContext(ur_context_handle_t Context); diff --git a/source/loader/layers/sanitizer/ur_sanddi.cpp b/source/loader/layers/sanitizer/ur_sanddi.cpp index d5b3360cd2..64f54752ca 100644 --- a/source/loader/layers/sanitizer/ur_sanddi.cpp +++ b/source/loader/layers/sanitizer/ur_sanddi.cpp @@ -181,24 +181,29 @@ __urdlllocal ur_result_t UR_APICALL urEnqueueKernelLaunch( return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } - ur_event_handle_t lk_event{}; + ur_event_handle_t hPreEvent{}; std::vector events(numEventsInWaitList + 1); for (unsigned i = 0; i < numEventsInWaitList; ++i) { events.push_back(phEventWaitList[i]); } // launchKernel must append to num_events_in_wait_list, not prepend - context.interceptor->preLaunchKernel(hKernel, hQueue, &lk_event); - if (lk_event) { - events.push_back(lk_event); + context.interceptor->preLaunchKernel(hKernel, hQueue, hPreEvent); + if (hPreEvent) { + events.push_back(hPreEvent); } + ur_event_handle_t hEvent{}; ur_result_t result = pfnKernelLaunch( hQueue, hKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, - pLocalWorkSize, numEventsInWaitList, phEventWaitList, phEvent); + pLocalWorkSize, numEventsInWaitList, phEventWaitList, &hEvent); if (result == UR_RESULT_SUCCESS) { - context.interceptor->postLaunchKernel(hKernel, hQueue, phEvent); + context.interceptor->postLaunchKernel(hKernel, hQueue, hEvent); + } + + if (phEvent) { + *phEvent = hEvent; } return result; From 577ddd3cc2396d782f3f8af907cc0ad916c12ebf Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Thu, 4 Jan 2024 17:51:52 -0800 Subject: [PATCH 70/72] fix build --- source/loader/layers/sanitizer/asan_interceptor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index 83051caea0..9a9c450571 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -322,7 +322,7 @@ ur_result_t SanitizerInterceptor::enqueueMemSetShadow( { static const size_t PageSize = [Context, Device]() { size_t Size; - auto Result = + [[maybe_unused]] auto Result = context.urDdiTable.VirtualMem.pfnGranularityGetInfo( Context, Device, UR_VIRTUAL_MEM_GRANULARITY_INFO_RECOMMENDED, From dc127d5ad7c6213c9acbe66e026b3f8926c299b6 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Mon, 8 Jan 2024 17:38:44 -0800 Subject: [PATCH 71/72] fix build --- source/loader/layers/sanitizer/asan_interceptor.cpp | 1 - source/loader/layers/sanitizer/asan_interceptor.hpp | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index 9a9c450571..2bd208a750 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -25,7 +25,6 @@ constexpr int kUsmDeviceRedzoneMagic = (char)0x81; constexpr int kUsmHostRedzoneMagic = (char)0x82; constexpr int kUsmSharedRedzoneMagic = (char)0x83; constexpr int kMemBufferRedzoneMagic = (char)0x84; -constexpr int kUnkownRedzoneMagic = (char)0x8F; constexpr auto kSPIR_AsanShadowMemoryGlobalStart = "__AsanShadowMemoryGlobalStart"; diff --git a/source/loader/layers/sanitizer/asan_interceptor.hpp b/source/loader/layers/sanitizer/asan_interceptor.hpp index a6838905d7..edad3f926e 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.hpp +++ b/source/loader/layers/sanitizer/asan_interceptor.hpp @@ -82,7 +82,6 @@ struct ContextInfo { class SanitizerInterceptor { public: - ur_result_t allocateMemory(ur_context_handle_t Context, ur_device_handle_t Device, const ur_usm_desc_t *Properties, @@ -99,11 +98,12 @@ class SanitizerInterceptor { ur_result_t eraseContext(ur_context_handle_t Context); ur_result_t insertDevice(ur_context_handle_t Context, - ur_device_handle_t Device); + ur_device_handle_t Device); - ur_result_t insertQueue(ur_context_handle_t Context, ur_queue_handle_t Queue); - ur_result_t eraseQueue(ur_context_handle_t Context, + ur_result_t insertQueue(ur_context_handle_t Context, ur_queue_handle_t Queue); + ur_result_t eraseQueue(ur_context_handle_t Context, + ur_queue_handle_t Queue); private: ur_result_t updateShadowMemory(ur_queue_handle_t Queue); From 28315db3cf33d438f73c337e3d9e5304444e16b9 Mon Sep 17 00:00:00 2001 From: "Zhao, Yang2" Date: Tue, 16 Jan 2024 18:11:03 -0800 Subject: [PATCH 72/72] fix build error --- .../layers/sanitizer/asan_interceptor.cpp | 3 ++- source/loader/layers/sanitizer/common.hpp | 18 +++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/source/loader/layers/sanitizer/asan_interceptor.cpp b/source/loader/layers/sanitizer/asan_interceptor.cpp index 2bd208a750..394405c056 100644 --- a/source/loader/layers/sanitizer/asan_interceptor.cpp +++ b/source/loader/layers/sanitizer/asan_interceptor.cpp @@ -420,7 +420,8 @@ ur_result_t SanitizerInterceptor::enqueueAllocInfo( auto Value = AllocInfo->UserEnd - RoundDownTo(AllocInfo->UserEnd, ASAN_SHADOW_GRANULARITY); UR_CALL(enqueueMemSetShadow(Context, Device, Queue, AllocInfo->UserEnd, - 1, Value, LastEvent, &LastEvent)); + 1, static_cast(Value), LastEvent, + &LastEvent)); } int ShadowByte; diff --git a/source/loader/layers/sanitizer/common.hpp b/source/loader/layers/sanitizer/common.hpp index e180f00a14..8b80814b9c 100644 --- a/source/loader/layers/sanitizer/common.hpp +++ b/source/loader/layers/sanitizer/common.hpp @@ -50,20 +50,20 @@ inline constexpr bool IsAligned(uptr a, uptr alignment) { // Valid redzone sizes are 16, 32, 64, ... 2048, so we encode them in 3 bits. // We use adaptive redzones: for larger allocation larger redzones are used. -inline constexpr u32 RZLog2Size(u32 rz_log) { +inline constexpr uptr RZLog2Size(uptr rz_log) { assert(rz_log < 8); return 16 << rz_log; } inline constexpr uptr ComputeRZLog(uptr user_requested_size) { - u32 rz_log = user_requested_size <= 64 - 16 ? 0 - : user_requested_size <= 128 - 32 ? 1 - : user_requested_size <= 512 - 64 ? 2 - : user_requested_size <= 4096 - 128 ? 3 - : user_requested_size <= (1 << 14) - 256 ? 4 - : user_requested_size <= (1 << 15) - 512 ? 5 - : user_requested_size <= (1 << 16) - 1024 ? 6 - : 7; + uptr rz_log = user_requested_size <= 64 - 16 ? 0 + : user_requested_size <= 128 - 32 ? 1 + : user_requested_size <= 512 - 64 ? 2 + : user_requested_size <= 4096 - 128 ? 3 + : user_requested_size <= (1 << 14) - 256 ? 4 + : user_requested_size <= (1 << 15) - 512 ? 5 + : user_requested_size <= (1 << 16) - 1024 ? 6 + : 7; return rz_log; }