diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 8f82ee0184..fdfaf6cc1f 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -85,7 +85,7 @@ jobs: - name: Test working-directory: ${{github.workspace}}/build - run: ctest -C ${{matrix.build_type}} --output-on-failure -L "python|uma|loader|validation|tracing|unit|urtrace" + run: ctest -C ${{matrix.build_type}} --output-on-failure -L "python|umf|loader|validation|tracing|unit|urtrace" windows-build: name: Build - Windows @@ -124,4 +124,4 @@ jobs: - name: Test working-directory: ${{github.workspace}}/build - run: ctest -C ${{matrix.build_type}} --output-on-failure -L "python|uma|loader|validation|tracing|unit|urtrace" + run: ctest -C ${{matrix.build_type}} --output-on-failure -L "python|umf|loader|validation|tracing|unit|urtrace" diff --git a/CMakeLists.txt b/CMakeLists.txt index b830166600..5836eb3fd0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,7 +26,7 @@ option(UR_FORMAT_CPP_STYLE "format code style of C++ sources" OFF) option(UR_USE_ASAN "enable AddressSanitizer" OFF) option(UR_USE_UBSAN "enable UndefinedBehaviorSanitizer" OFF) option(UR_USE_MSAN "enable MemorySanitizer" OFF) -option(UMA_BUILD_SHARED_LIBRARY "Build UMA as shared library" OFF) +option(UMF_BUILD_SHARED_LIBRARY "Build UMF as shared library" OFF) option(UR_ENABLE_TRACING "enable api tracing through xpti" OFF) option(VAL_USE_LIBBACKTRACE_BACKTRACE "enable libbacktrace validation backtrace for linux" OFF) option(UR_BUILD_TOOLS "build ur tools" ON) diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index aefc0c6195..f240f9908b 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -11,20 +11,20 @@ target_include_directories(ur_common INTERFACE ${CMAKE_SOURCE_DIR}/include ) -add_subdirectory(unified_memory_allocation) -add_subdirectory(uma_pools) -target_link_libraries(ur_common INTERFACE unified_memory_allocation disjoint_pool ${CMAKE_DL_LIBS} ${PROJECT_NAME}::headers) +add_subdirectory(unified_malloc_framework) +add_subdirectory(umf_pools) +target_link_libraries(ur_common INTERFACE unified_malloc_framework disjoint_pool ${CMAKE_DL_LIBS} ${PROJECT_NAME}::headers) if(WIN32) target_sources(ur_common INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/windows/ur_lib_loader.cpp - uma_helpers.hpp ur_pool_manager.hpp + umf_helpers.hpp ur_pool_manager.hpp ) else() target_sources(ur_common INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/linux/ur_lib_loader.cpp - uma_helpers.hpp ur_pool_manager.hpp + umf_helpers.hpp ur_pool_manager.hpp ) endif() diff --git a/source/common/uma_helpers.hpp b/source/common/umf_helpers.hpp similarity index 54% rename from source/common/uma_helpers.hpp rename to source/common/umf_helpers.hpp index 95a9c04f05..510af3e244 100644 --- a/source/common/uma_helpers.hpp +++ b/source/common/umf_helpers.hpp @@ -8,13 +8,13 @@ * */ -#ifndef UMA_HELPERS_H -#define UMA_HELPERS_H 1 +#ifndef UMF_HELPERS_H +#define UMF_HELPERS_H 1 -#include -#include -#include -#include +#include +#include +#include +#include #include #include @@ -22,16 +22,16 @@ #include #include -namespace uma { +namespace umf { using pool_unique_handle_t = - std::unique_ptr>; + std::unique_ptr>; using provider_unique_handle_t = - std::unique_ptr>; + std::unique_ptr>; -#define UMA_ASSIGN_OP(ops, type, func, default_return) \ +#define UMF_ASSIGN_OP(ops, type, func, default_return) \ ops.func = [](void *obj, auto... args) { \ try { \ return reinterpret_cast(obj)->func(args...); \ @@ -40,7 +40,7 @@ using provider_unique_handle_t = } \ } -#define UMA_ASSIGN_OP_NORETURN(ops, type, func) \ +#define UMF_ASSIGN_OP_NORETURN(ops, type, func) \ ops.func = [](void *obj, auto... args) { \ try { \ return reinterpret_cast(obj)->func(args...); \ @@ -48,24 +48,24 @@ using provider_unique_handle_t = } \ } -/// @brief creates UMA memory provider based on given T type. +/// @brief creates UMF memory provider based on given T type. /// T should implement all functions defined by -/// uma_memory_provider_ops_t, except for finalize (it is +/// umf_memory_provider_ops_t, except for finalize (it is /// replaced by dtor). All arguments passed to this function are /// forwarded to T::initialize(). template auto memoryProviderMakeUnique(Args &&...args) { - uma_memory_provider_ops_t ops; + umf_memory_provider_ops_t ops; auto argsTuple = std::make_tuple(std::forward(args)...); - ops.version = UMA_VERSION_CURRENT; + ops.version = UMF_VERSION_CURRENT; ops.initialize = [](void *params, void **obj) { auto *tuple = reinterpret_cast(params); T *provider; try { provider = new T; } catch (...) { - return UMA_RESULT_ERROR_OUT_OF_HOST_MEMORY; + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } *obj = provider; @@ -74,45 +74,45 @@ auto memoryProviderMakeUnique(Args &&...args) { auto ret = std::apply(&T::initialize, std::tuple_cat(std::make_tuple(provider), *tuple)); - if (ret != UMA_RESULT_SUCCESS) { + if (ret != UMF_RESULT_SUCCESS) { delete provider; } return ret; } catch (...) { delete provider; - return UMA_RESULT_ERROR_UNKNOWN; + return UMF_RESULT_ERROR_UNKNOWN; } }; ops.finalize = [](void *obj) { delete reinterpret_cast(obj); }; - UMA_ASSIGN_OP(ops, T, alloc, UMA_RESULT_ERROR_UNKNOWN); - UMA_ASSIGN_OP(ops, T, free, UMA_RESULT_ERROR_UNKNOWN); - UMA_ASSIGN_OP_NORETURN(ops, T, get_last_native_error); - UMA_ASSIGN_OP(ops, T, get_recommended_page_size, UMA_RESULT_ERROR_UNKNOWN); - UMA_ASSIGN_OP(ops, T, get_min_page_size, UMA_RESULT_ERROR_UNKNOWN); - UMA_ASSIGN_OP(ops, T, purge_lazy, UMA_RESULT_ERROR_UNKNOWN); - UMA_ASSIGN_OP(ops, T, purge_force, UMA_RESULT_ERROR_UNKNOWN); - UMA_ASSIGN_OP(ops, T, get_name, ""); - - uma_memory_provider_handle_t hProvider = nullptr; - auto ret = umaMemoryProviderCreate(&ops, &argsTuple, &hProvider); - return std::pair{ - ret, provider_unique_handle_t(hProvider, &umaMemoryProviderDestroy)}; + UMF_ASSIGN_OP(ops, T, alloc, UMF_RESULT_ERROR_UNKNOWN); + UMF_ASSIGN_OP(ops, T, free, UMF_RESULT_ERROR_UNKNOWN); + UMF_ASSIGN_OP_NORETURN(ops, T, get_last_native_error); + UMF_ASSIGN_OP(ops, T, get_recommended_page_size, UMF_RESULT_ERROR_UNKNOWN); + UMF_ASSIGN_OP(ops, T, get_min_page_size, UMF_RESULT_ERROR_UNKNOWN); + UMF_ASSIGN_OP(ops, T, purge_lazy, UMF_RESULT_ERROR_UNKNOWN); + UMF_ASSIGN_OP(ops, T, purge_force, UMF_RESULT_ERROR_UNKNOWN); + UMF_ASSIGN_OP(ops, T, get_name, ""); + + umf_memory_provider_handle_t hProvider = nullptr; + auto ret = umfMemoryProviderCreate(&ops, &argsTuple, &hProvider); + return std::pair{ + ret, provider_unique_handle_t(hProvider, &umfMemoryProviderDestroy)}; } -/// @brief creates UMA memory pool based on given T type. +/// @brief creates UMF memory pool based on given T type. /// T should implement all functions defined by -/// uma_memory_provider_ops_t, except for finalize (it is +/// umf_memory_provider_ops_t, except for finalize (it is /// replaced by dtor). All arguments passed to this function are /// forwarded to T::initialize(). template -auto poolMakeUnique(uma_memory_provider_handle_t *providers, +auto poolMakeUnique(umf_memory_provider_handle_t *providers, size_t numProviders, Args &&...args) { - uma_memory_pool_ops_t ops; + umf_memory_pool_ops_t ops; auto argsTuple = std::make_tuple(std::forward(args)...); - ops.version = UMA_VERSION_CURRENT; - ops.initialize = [](uma_memory_provider_handle_t *providers, + ops.version = UMF_VERSION_CURRENT; + ops.initialize = [](umf_memory_provider_handle_t *providers, size_t numProviders, void *params, void **obj) { auto *tuple = reinterpret_cast(params); T *pool; @@ -120,7 +120,7 @@ auto poolMakeUnique(uma_memory_provider_handle_t *providers, try { pool = new T; } catch (...) { - return UMA_RESULT_ERROR_OUT_OF_HOST_MEMORY; + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } *obj = pool; @@ -130,36 +130,36 @@ auto poolMakeUnique(uma_memory_provider_handle_t *providers, &T::initialize, std::tuple_cat(std::make_tuple(pool, providers, numProviders), *tuple)); - if (ret != UMA_RESULT_SUCCESS) { + if (ret != UMF_RESULT_SUCCESS) { delete pool; } return ret; } catch (...) { delete pool; - return UMA_RESULT_ERROR_UNKNOWN; + return UMF_RESULT_ERROR_UNKNOWN; } }; ops.finalize = [](void *obj) { delete reinterpret_cast(obj); }; - UMA_ASSIGN_OP(ops, T, malloc, ((void *)nullptr)); - UMA_ASSIGN_OP(ops, T, calloc, ((void *)nullptr)); - UMA_ASSIGN_OP(ops, T, aligned_malloc, ((void *)nullptr)); - UMA_ASSIGN_OP(ops, T, realloc, ((void *)nullptr)); - UMA_ASSIGN_OP(ops, T, malloc_usable_size, ((size_t)0)); - UMA_ASSIGN_OP_NORETURN(ops, T, free); - UMA_ASSIGN_OP(ops, T, get_last_allocation_error, UMA_RESULT_ERROR_UNKNOWN); - - uma_memory_pool_handle_t hPool = nullptr; - auto ret = umaPoolCreate(&ops, providers, numProviders, &argsTuple, &hPool); - return std::pair{ - ret, pool_unique_handle_t(hPool, &umaPoolDestroy)}; + UMF_ASSIGN_OP(ops, T, malloc, ((void *)nullptr)); + UMF_ASSIGN_OP(ops, T, calloc, ((void *)nullptr)); + UMF_ASSIGN_OP(ops, T, aligned_malloc, ((void *)nullptr)); + UMF_ASSIGN_OP(ops, T, realloc, ((void *)nullptr)); + UMF_ASSIGN_OP(ops, T, malloc_usable_size, ((size_t)0)); + UMF_ASSIGN_OP_NORETURN(ops, T, free); + UMF_ASSIGN_OP(ops, T, get_last_allocation_error, UMF_RESULT_ERROR_UNKNOWN); + + umf_memory_pool_handle_t hPool = nullptr; + auto ret = umfPoolCreate(&ops, providers, numProviders, &argsTuple, &hPool); + return std::pair{ + ret, pool_unique_handle_t(hPool, &umfPoolDestroy)}; } -template uma_result_t &getPoolLastStatusRef() { - static thread_local uma_result_t last_status = UMA_RESULT_SUCCESS; +template umf_result_t &getPoolLastStatusRef() { + static thread_local umf_result_t last_status = UMF_RESULT_SUCCESS; return last_status; } -} // namespace uma +} // namespace umf -#endif /* UMA_HELPERS_H */ +#endif /* UMF_HELPERS_H */ diff --git a/source/common/uma_pools/CMakeLists.txt b/source/common/umf_pools/CMakeLists.txt similarity index 96% rename from source/common/uma_pools/CMakeLists.txt rename to source/common/umf_pools/CMakeLists.txt index cd93cad0a9..fc511cd814 100644 --- a/source/common/uma_pools/CMakeLists.txt +++ b/source/common/umf_pools/CMakeLists.txt @@ -11,7 +11,7 @@ add_library(disjoint_pool STATIC add_library(${PROJECT_NAME}::disjoint_pool ALIAS disjoint_pool) target_link_libraries(disjoint_pool PRIVATE - unified_memory_allocation + unified_malloc_framework ${PROJECT_NAME}::headers) if (UNIX) diff --git a/source/common/uma_pools/disjoint_pool.cpp b/source/common/umf_pools/disjoint_pool.cpp similarity index 96% rename from source/common/uma_pools/disjoint_pool.cpp rename to source/common/umf_pools/disjoint_pool.cpp index 34f72a290e..e4c4c693dd 100644 --- a/source/common/uma_pools/disjoint_pool.cpp +++ b/source/common/umf_pools/disjoint_pool.cpp @@ -75,7 +75,7 @@ static size_t AlignUp(size_t Val, size_t Alignment) { } struct MemoryProviderError { - uma_result_t code; + umf_result_t code; }; DisjointPoolConfig::DisjointPoolConfig() @@ -220,7 +220,7 @@ class Bucket { // Free an allocation that is a full slab in this bucket. void freeSlab(Slab &Slab, bool &ToPool); - uma_memory_provider_handle_t getMemHandle(); + umf_memory_provider_handle_t getMemHandle(); DisjointPool::AllocImpl &getAllocCtx() { return OwnAllocCtx; } @@ -275,7 +275,7 @@ class DisjointPool::AllocImpl { std::shared_timed_mutex KnownSlabsMapLock; // Handle to the memory provider - uma_memory_provider_handle_t MemHandle; + umf_memory_provider_handle_t MemHandle; // Store as unique_ptrs since Bucket is not Movable(because of std::mutex) std::vector> Buckets; @@ -287,7 +287,7 @@ class DisjointPool::AllocImpl { size_t ProviderMinPageSize; public: - AllocImpl(uma_memory_provider_handle_t hProvider, DisjointPoolConfig params) + AllocImpl(umf_memory_provider_handle_t hProvider, DisjointPoolConfig params) : MemHandle{hProvider}, params(params) { // Generate buckets sized such as: 64, 96, 128, 192, ..., CutOff. @@ -300,9 +300,9 @@ class DisjointPool::AllocImpl { } Buckets.push_back(std::make_unique(CutOff, *this)); - auto ret = umaMemoryProviderGetMinPageSize(hProvider, nullptr, + auto ret = umfMemoryProviderGetMinPageSize(hProvider, nullptr, &ProviderMinPageSize); - if (ret != UMA_RESULT_SUCCESS) { + if (ret != UMF_RESULT_SUCCESS) { ProviderMinPageSize = 0; } } @@ -311,7 +311,7 @@ class DisjointPool::AllocImpl { void *allocate(size_t Size, bool &FromPool); void deallocate(void *Ptr, bool &ToPool); - uma_memory_provider_handle_t getMemHandle() { return MemHandle; } + umf_memory_provider_handle_t getMemHandle() { return MemHandle; } std::shared_timed_mutex &getKnownSlabsMapLock() { return KnownSlabsMapLock; @@ -331,20 +331,20 @@ class DisjointPool::AllocImpl { Bucket &findBucket(size_t Size); }; -static void *memoryProviderAlloc(uma_memory_provider_handle_t hProvider, +static void *memoryProviderAlloc(umf_memory_provider_handle_t hProvider, size_t size, size_t alignment = 0) { void *ptr; - auto ret = umaMemoryProviderAlloc(hProvider, size, alignment, &ptr); - if (ret != UMA_RESULT_SUCCESS) { + auto ret = umfMemoryProviderAlloc(hProvider, size, alignment, &ptr); + if (ret != UMF_RESULT_SUCCESS) { throw MemoryProviderError{ret}; } return ptr; } -static void memoryProviderFree(uma_memory_provider_handle_t hProvider, +static void memoryProviderFree(umf_memory_provider_handle_t hProvider, void *ptr) { - auto ret = umaMemoryProviderFree(hProvider, ptr, 0); - if (ret != UMA_RESULT_SUCCESS) { + auto ret = umfMemoryProviderFree(hProvider, ptr, 0); + if (ret != UMF_RESULT_SUCCESS) { throw MemoryProviderError{ret}; } } @@ -663,7 +663,7 @@ bool Bucket::CanPool(bool &ToPool) { return false; } -uma_memory_provider_handle_t Bucket::getMemHandle() { +umf_memory_provider_handle_t Bucket::getMemHandle() { return OwnAllocCtx.getMemHandle(); } @@ -753,7 +753,7 @@ void *DisjointPool::AllocImpl::allocate(size_t Size, bool &FromPool) try { return Ptr; } catch (MemoryProviderError &e) { - uma::getPoolLastStatusRef() = e.code; + umf::getPoolLastStatusRef() = e.code; return nullptr; } @@ -803,7 +803,7 @@ void *DisjointPool::AllocImpl::allocate(size_t Size, size_t Alignment, return AlignPtrUp(Ptr, Alignment); } catch (MemoryProviderError &e) { - uma::getPoolLastStatusRef() = e.code; + umf::getPoolLastStatusRef() = e.code; return nullptr; } @@ -863,7 +863,7 @@ void DisjointPool::AllocImpl::deallocate(void *Ptr, bool &ToPool) try { // but the range checks fail. memoryProviderFree(getMemHandle(), Ptr); } catch (MemoryProviderError &e) { - uma::getPoolLastStatusRef() = e.code; + umf::getPoolLastStatusRef() = e.code; } void DisjointPool::AllocImpl::printStats(bool &TitlePrinted, @@ -881,15 +881,15 @@ void DisjointPool::AllocImpl::printStats(bool &TitlePrinted, } } -uma_result_t DisjointPool::initialize(uma_memory_provider_handle_t *providers, +umf_result_t DisjointPool::initialize(umf_memory_provider_handle_t *providers, size_t numProviders, DisjointPoolConfig parameters) { if (numProviders != 1 || !providers[0]) { - return UMA_RESULT_ERROR_INVALID_ARGUMENT; + return UMF_RESULT_ERROR_INVALID_ARGUMENT; } impl = std::make_unique(providers[0], parameters); - return UMA_RESULT_SUCCESS; + return UMF_RESULT_SUCCESS; } void *DisjointPool::malloc(size_t size) { // For full-slab allocations indicates @@ -955,8 +955,8 @@ void DisjointPool::free(void *ptr) { return; } -enum uma_result_t DisjointPool::get_last_allocation_error() { - return uma::getPoolLastStatusRef(); +enum umf_result_t DisjointPool::get_last_allocation_error() { + return umf::getPoolLastStatusRef(); } DisjointPool::DisjointPool() {} diff --git a/source/common/uma_pools/disjoint_pool.hpp b/source/common/umf_pools/disjoint_pool.hpp similarity index 93% rename from source/common/uma_pools/disjoint_pool.hpp rename to source/common/umf_pools/disjoint_pool.hpp index fb1e34f2d7..85dd86872d 100644 --- a/source/common/uma_pools/disjoint_pool.hpp +++ b/source/common/umf_pools/disjoint_pool.hpp @@ -12,7 +12,7 @@ #include #include -#include "../uma_helpers.hpp" +#include "../umf_helpers.hpp" namespace usm { @@ -60,7 +60,7 @@ class DisjointPool { class AllocImpl; using Config = DisjointPoolConfig; - uma_result_t initialize(uma_memory_provider_handle_t *providers, + umf_result_t initialize(umf_memory_provider_handle_t *providers, size_t numProviders, DisjointPoolConfig parameters); void *malloc(size_t size); void *calloc(size_t, size_t); @@ -68,7 +68,7 @@ class DisjointPool { void *aligned_malloc(size_t size, size_t alignment); size_t malloc_usable_size(void *); void free(void *ptr); - enum uma_result_t get_last_allocation_error(); + enum umf_result_t get_last_allocation_error(); DisjointPool(); ~DisjointPool(); diff --git a/source/common/uma_pools/disjoint_pool_config_parser.cpp b/source/common/umf_pools/disjoint_pool_config_parser.cpp similarity index 100% rename from source/common/uma_pools/disjoint_pool_config_parser.cpp rename to source/common/umf_pools/disjoint_pool_config_parser.cpp diff --git a/source/common/uma_pools/disjoint_pool_config_parser.hpp b/source/common/umf_pools/disjoint_pool_config_parser.hpp similarity index 100% rename from source/common/uma_pools/disjoint_pool_config_parser.hpp rename to source/common/umf_pools/disjoint_pool_config_parser.hpp diff --git a/source/common/unified_memory_allocation/CMakeLists.txt b/source/common/unified_malloc_framework/CMakeLists.txt similarity index 55% rename from source/common/unified_memory_allocation/CMakeLists.txt rename to source/common/unified_malloc_framework/CMakeLists.txt index 7be1ab2dce..30991629cd 100644 --- a/source/common/unified_memory_allocation/CMakeLists.txt +++ b/source/common/unified_malloc_framework/CMakeLists.txt @@ -3,24 +3,24 @@ # See LICENSE.TXT # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -set(UMA_SOURCES +set(UMF_SOURCES src/memory_pool.c src/memory_provider.c src/memory_tracker.cpp src/memory_provider_get_last_failed.cpp ) -if(UMA_BUILD_SHARED_LIBRARY) - message(WARNING "Unified Memory Allocation is still an early work in progress." +if(UMF_BUILD_SHARED_LIBRARY) + message(WARNING "Unified Malloc Framework is still an early work in progress." "There are no API/ABI backward compatibility guarantees. There will be breakages." "Do not use the shared library in production software.") - add_library(unified_memory_allocation SHARED - ${UMA_SOURCES}) + add_library(unified_malloc_framework SHARED + ${UMF_SOURCES}) else() - add_library(unified_memory_allocation STATIC - ${UMA_SOURCES}) + add_library(unified_malloc_framework STATIC + ${UMF_SOURCES}) endif() -add_library(${PROJECT_NAME}::unified_memory_allocation ALIAS unified_memory_allocation) +add_library(${PROJECT_NAME}::unified_malloc_framework ALIAS unified_malloc_framework) -target_include_directories(unified_memory_allocation PUBLIC include) +target_include_directories(unified_malloc_framework PUBLIC include) diff --git a/source/common/unified_memory_allocation/include/uma.h b/source/common/unified_malloc_framework/include/umf.h similarity index 54% rename from source/common/unified_memory_allocation/include/uma.h rename to source/common/unified_malloc_framework/include/umf.h index 9b6eb1a435..c46eaa3e1f 100644 --- a/source/common/unified_memory_allocation/include/uma.h +++ b/source/common/unified_malloc_framework/include/umf.h @@ -8,10 +8,10 @@ * */ -#ifndef UMA_UNIFIED_MEMORY_ALLOCATION_H -#define UMA_UNIFIED_MEMORY_ALLOCATION_H 1 +#ifndef UMF_UNIFIED_MEMORY_ALLOCATION_H +#define UMF_UNIFIED_MEMORY_ALLOCATION_H 1 -#include -#include +#include +#include -#endif /* UMA_UNIFIED_MEMORY_ALLOCATION_H */ +#endif /* UMF_UNIFIED_MEMORY_ALLOCATION_H */ diff --git a/source/common/unified_malloc_framework/include/umf/base.h b/source/common/unified_malloc_framework/include/umf/base.h new file mode 100644 index 0000000000..6971bc0cab --- /dev/null +++ b/source/common/unified_malloc_framework/include/umf/base.h @@ -0,0 +1,54 @@ +/* + * + * Copyright (C) 2023 Intel Corporation + * + * Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. + * See LICENSE.TXT + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#ifndef UMF_BASE_H +#define UMF_BASE_H 1 + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/// \brief Generates generic 'UMF' API versions +#define UMF_MAKE_VERSION(_major, _minor) \ + ((_major << 16) | (_minor & 0x0000ffff)) + +/// \brief Extracts 'UMF' API major version +#define UMF_MAJOR_VERSION(_ver) (_ver >> 16) + +/// \brief Extracts 'UMF' API minor version +#define UMF_MINOR_VERSION(_ver) (_ver & 0x0000ffff) + +/// \brief Current version of the UMF headers +#define UMF_VERSION_CURRENT UMF_MAKE_VERSION(0, 9) + +/// \brief Operation results +enum umf_result_t { + UMF_RESULT_SUCCESS = 0, ///< Success + UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY = + 1, ///< Insufficient host memory to satisfy call, + UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC = + 2, ///< A provider specific warning/error has been reported and can be + ///< Retrieved via the umfMemoryProviderGetLastNativeError entry point. + UMF_RESULT_ERROR_INVALID_ARGUMENT = + 3, ///< Generic error code for invalid arguments + UMF_RESULT_ERROR_INVALID_ALIGNMENT = 4, /// Invalid alignment of an argument + UMF_RESULT_ERROR_NOT_SUPPORTED = 5, /// Operation not supported + + UMF_RESULT_ERROR_UNKNOWN = 0x7ffffffe ///< Unknown or internal error +}; + +#ifdef __cplusplus +} +#endif + +#endif /* UMF_BASE_H */ diff --git a/source/common/unified_memory_allocation/include/uma/memory_pool.h b/source/common/unified_malloc_framework/include/umf/memory_pool.h similarity index 72% rename from source/common/unified_memory_allocation/include/uma/memory_pool.h rename to source/common/unified_malloc_framework/include/umf/memory_pool.h index 812aa0c963..294ce6a46c 100644 --- a/source/common/unified_memory_allocation/include/uma/memory_pool.h +++ b/source/common/unified_malloc_framework/include/umf/memory_pool.h @@ -8,40 +8,40 @@ * */ -#ifndef UMA_MEMORY_POOL_H -#define UMA_MEMORY_POOL_H 1 +#ifndef UMF_MEMORY_POOL_H +#define UMF_MEMORY_POOL_H 1 -#include -#include +#include +#include #ifdef __cplusplus extern "C" { #endif -typedef struct uma_memory_pool_t *uma_memory_pool_handle_t; +typedef struct umf_memory_pool_t *umf_memory_pool_handle_t; -struct uma_memory_pool_ops_t; +struct umf_memory_pool_ops_t; /// /// \brief Creates new memory pool. -/// \param ops instance of uma_memory_pool_ops_t +/// \param ops instance of umf_memory_pool_ops_t /// \param providers array of memory providers that will be used for coarse-grain allocations. /// Should contain at least one memory provider. /// \param numProvider number of elements in the providers array /// \param params pointer to pool-specific parameters /// \param hPool [out] handle to the newly created memory pool -/// \return UMA_RESULT_SUCCESS on success or appropriate error code on failure. +/// \return UMF_RESULT_SUCCESS on success or appropriate error code on failure. /// -enum uma_result_t umaPoolCreate(struct uma_memory_pool_ops_t *ops, - uma_memory_provider_handle_t *providers, +enum umf_result_t umfPoolCreate(struct umf_memory_pool_ops_t *ops, + umf_memory_provider_handle_t *providers, size_t numProviders, void *params, - uma_memory_pool_handle_t *hPool); + umf_memory_pool_handle_t *hPool); /// /// \brief Destroys memory pool. /// \param hPool handle to the pool /// -void umaPoolDestroy(uma_memory_pool_handle_t hPool); +void umfPoolDestroy(umf_memory_pool_handle_t hPool); /// /// \brief Allocates size bytes of uninitialized storage of the specified hPool. @@ -49,7 +49,7 @@ void umaPoolDestroy(uma_memory_pool_handle_t hPool); /// \param size number of bytes to allocate /// \return Pointer to the allocated memory. /// -void *umaPoolMalloc(uma_memory_pool_handle_t hPool, size_t size); +void *umfPoolMalloc(umf_memory_pool_handle_t hPool, size_t size); /// /// \brief Allocates size bytes of uninitialized storage of the specified hPool. @@ -59,7 +59,7 @@ void *umaPoolMalloc(uma_memory_pool_handle_t hPool, size_t size); /// \param alignment alignment of the allocation /// \return Pointer to the allocated memory. /// -void *umaPoolAlignedMalloc(uma_memory_pool_handle_t hPool, size_t size, +void *umfPoolAlignedMalloc(umf_memory_pool_handle_t hPool, size_t size, size_t alignment); /// @@ -71,7 +71,7 @@ void *umaPoolAlignedMalloc(uma_memory_pool_handle_t hPool, size_t size, /// \param size specified size of each element /// \return Pointer to the allocated memory. /// -void *umaPoolCalloc(uma_memory_pool_handle_t hPool, size_t num, size_t size); +void *umfPoolCalloc(umf_memory_pool_handle_t hPool, size_t num, size_t size); /// /// \brief Reallocates memory of the specified hPool. @@ -80,7 +80,7 @@ void *umaPoolCalloc(uma_memory_pool_handle_t hPool, size_t num, size_t size); /// \param size new size for the memory block in bytes /// \return Pointer to the allocated memory. /// -void *umaPoolRealloc(uma_memory_pool_handle_t hPool, void *ptr, size_t size); +void *umfPoolRealloc(umf_memory_pool_handle_t hPool, void *ptr, size_t size); /// /// \brief Obtains size of block of memory allocated from the pool. @@ -88,23 +88,23 @@ void *umaPoolRealloc(uma_memory_pool_handle_t hPool, void *ptr, size_t size); /// \param ptr pointer to the allocated memory /// \return Number of bytes. /// -size_t umaPoolMallocUsableSize(uma_memory_pool_handle_t hPool, void *ptr); +size_t umfPoolMallocUsableSize(umf_memory_pool_handle_t hPool, void *ptr); /// /// \brief Frees the memory space of the specified hPool pointed by ptr. /// \param hPool specified memory hPool /// \param ptr pointer to the allocated memory /// -void umaPoolFree(uma_memory_pool_handle_t hPool, void *ptr); +void umfPoolFree(umf_memory_pool_handle_t hPool, void *ptr); /// -/// \brief Frees the memory space pointed by ptr if it belongs to UMA pool, does nothing otherwise. +/// \brief Frees the memory space pointed by ptr if it belongs to UMF pool, does nothing otherwise. /// \param ptr pointer to the allocated memory /// -void umaFree(void *ptr); +void umfFree(void *ptr); /// -/// \brief Retrieve uma_result_t representing the error of the last failed allocation +/// \brief Retrieve umf_result_t representing the error of the last failed allocation /// operation in this thread (malloc, calloc, realloc, aligned_malloc). /// /// \details @@ -120,29 +120,29 @@ void umaFree(void *ptr); /// \param hPool specified memory hPool /// \return Error code desciribng the failure of the last failed allocation operation. /// The value is undefined if the previous allocation was successful. -enum uma_result_t umaPoolGetLastAllocationError(uma_memory_pool_handle_t hPool); +enum umf_result_t umfPoolGetLastAllocationError(umf_memory_pool_handle_t hPool); /// /// \brief Retrieve memory pool associated with a given ptr. /// \param ptr pointer to memory belonging to a memory pool -/// \return Handle to a memory pool that contains ptr or NULL if pointer does not belong to any UMA pool. -uma_memory_pool_handle_t umaPoolByPtr(const void *ptr); +/// \return Handle to a memory pool that contains ptr or NULL if pointer does not belong to any UMF pool. +umf_memory_pool_handle_t umfPoolByPtr(const void *ptr); /// /// \brief Retrieve memory providers associated with a given pool. /// \param hPool specified memory pool /// \param hProviders [out] pointer to an array of memory providers. If numProviders is not equal to or -/// greater than the real number of providers, UMA_RESULT_ERROR_INVALID_ARGUMENT is returned. +/// greater than the real number of providers, UMF_RESULT_ERROR_INVALID_ARGUMENT is returned. /// \param numProviders [in] number of memory providers to return /// \param numProvidersRet pointer to the actual number of memory providers -/// \return UMA_RESULT_SUCCESS on success or appropriate error code on failure. -enum uma_result_t -umaPoolGetMemoryProviders(uma_memory_pool_handle_t hPool, size_t numProviders, - uma_memory_provider_handle_t *hProviders, +/// \return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +enum umf_result_t +umfPoolGetMemoryProviders(umf_memory_pool_handle_t hPool, size_t numProviders, + umf_memory_provider_handle_t *hProviders, size_t *numProvidersRet); #ifdef __cplusplus } #endif -#endif /* UMA_MEMORY_POOL_H */ +#endif /* UMF_MEMORY_POOL_H */ diff --git a/source/common/unified_memory_allocation/include/uma/memory_pool_ops.h b/source/common/unified_malloc_framework/include/umf/memory_pool_ops.h similarity index 78% rename from source/common/unified_memory_allocation/include/uma/memory_pool_ops.h rename to source/common/unified_malloc_framework/include/umf/memory_pool_ops.h index 21fae70b27..ce7aa960d7 100644 --- a/source/common/unified_memory_allocation/include/uma/memory_pool_ops.h +++ b/source/common/unified_malloc_framework/include/umf/memory_pool_ops.h @@ -8,21 +8,21 @@ * */ -#ifndef UMA_MEMORY_POOL_OPS_H -#define UMA_MEMORY_POOL_OPS_H 1 +#ifndef UMF_MEMORY_POOL_OPS_H +#define UMF_MEMORY_POOL_OPS_H 1 -#include +#include #ifdef __cplusplus extern "C" { #endif -/// \brief This structure comprises function pointers used by corresponding umaPool* +/// \brief This structure comprises function pointers used by corresponding umfPool* /// calls. Each memory pool implementation should initialize all function /// pointers. -struct uma_memory_pool_ops_t { +struct umf_memory_pool_ops_t { /// Version of the ops structure. - /// Should be initialized using UMA_VERSION_CURRENT + /// Should be initialized using UMF_VERSION_CURRENT uint32_t version; /// @@ -32,8 +32,8 @@ struct uma_memory_pool_ops_t { /// \param numProvider number of elements in the providers array /// \param params pool-specific params /// \param pool [out] returns pointer to the pool - /// \return UMA_RESULT_SUCCESS on success or appropriate error code on failure. - enum uma_result_t (*initialize)(uma_memory_provider_handle_t *providers, + /// \return UMF_RESULT_SUCCESS on success or appropriate error code on failure. + enum umf_result_t (*initialize)(umf_memory_provider_handle_t *providers, size_t numProviders, void *params, void **pool); @@ -49,11 +49,11 @@ struct uma_memory_pool_ops_t { void *(*aligned_malloc)(void *pool, size_t size, size_t alignment); size_t (*malloc_usable_size)(void *pool, void *ptr); void (*free)(void *pool, void *); - enum uma_result_t (*get_last_allocation_error)(void *pool); + enum umf_result_t (*get_last_allocation_error)(void *pool); }; #ifdef __cplusplus } #endif -#endif /* UMA_MEMORY_POOL_OPS_H */ +#endif /* UMF_MEMORY_POOL_OPS_H */ diff --git a/source/common/unified_memory_allocation/include/uma/memory_provider.h b/source/common/unified_malloc_framework/include/umf/memory_provider.h similarity index 68% rename from source/common/unified_memory_allocation/include/uma/memory_provider.h rename to source/common/unified_malloc_framework/include/umf/memory_provider.h index 9b65dc6419..8a911a62a3 100644 --- a/source/common/unified_memory_allocation/include/uma/memory_provider.h +++ b/source/common/unified_malloc_framework/include/umf/memory_provider.h @@ -8,34 +8,34 @@ * */ -#ifndef UMA_MEMORY_PROVIDER_H -#define UMA_MEMORY_PROVIDER_H 1 +#ifndef UMF_MEMORY_PROVIDER_H +#define UMF_MEMORY_PROVIDER_H 1 -#include -#include +#include +#include #ifdef __cplusplus extern "C" { #endif -typedef struct uma_memory_provider_t *uma_memory_provider_handle_t; +typedef struct umf_memory_provider_t *umf_memory_provider_handle_t; /// /// \brief Creates new memory provider. -/// \param ops instance of uma_memory_provider_ops_t +/// \param ops instance of umf_memory_provider_ops_t /// \param params pointer to provider-specific parameters /// \param hProvider [out] pointer to the newly created memory provider -/// \return UMA_RESULT_SUCCESS on success or appropriate error code on failure. +/// \return UMF_RESULT_SUCCESS on success or appropriate error code on failure. /// -enum uma_result_t -umaMemoryProviderCreate(struct uma_memory_provider_ops_t *ops, void *params, - uma_memory_provider_handle_t *hProvider); +enum umf_result_t +umfMemoryProviderCreate(struct umf_memory_provider_ops_t *ops, void *params, + umf_memory_provider_handle_t *hProvider); /// /// \brief Destroys memory provider. /// \param hPool handle to the memory provider /// -void umaMemoryProviderDestroy(uma_memory_provider_handle_t hProvider); +void umfMemoryProviderDestroy(umf_memory_provider_handle_t hProvider); /// /// \brief Allocates size bytes of uninitialized storage from memory provider @@ -44,9 +44,9 @@ void umaMemoryProviderDestroy(uma_memory_provider_handle_t hProvider); /// \param size number of bytes to allocate /// \param alignment alignment of the allocation /// \param ptr [out] pointer to the allocated memory -/// \return UMA_RESULT_SUCCESS on success or appropriate error code on failure. +/// \return UMF_RESULT_SUCCESS on success or appropriate error code on failure. /// -enum uma_result_t umaMemoryProviderAlloc(uma_memory_provider_handle_t hProvider, +enum umf_result_t umfMemoryProviderAlloc(umf_memory_provider_handle_t hProvider, size_t size, size_t alignment, void **ptr); @@ -55,23 +55,23 @@ enum uma_result_t umaMemoryProviderAlloc(uma_memory_provider_handle_t hProvider, /// \param hProvider handle to the memory provider /// \param ptr pointer to the allocated memory /// \param size size of the allocation -/// \return UMA_RESULT_SUCCESS on success or appropriate error code on failure. +/// \return UMF_RESULT_SUCCESS on success or appropriate error code on failure. /// -enum uma_result_t umaMemoryProviderFree(uma_memory_provider_handle_t hProvider, +enum umf_result_t umfMemoryProviderFree(umf_memory_provider_handle_t hProvider, void *ptr, size_t size); /// /// \brief Retrieve string representation of the underlying provider specific /// result reported by the last API that returned -/// UMA_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC. Allows for a provider +/// UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC. Allows for a provider /// independent way to return a provider specific result. /// /// \details /// * Implementations *must* store the message and error code in thread-local -/// storage prior to returning UMA_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC. +/// storage prior to returning UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC. /// /// * The message and error code will only be valid if a previously -/// called entry-point returned UMA_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC. +/// called entry-point returned UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC. /// /// * The memory pointed to by the C string returned in `ppMessage` is owned by /// the adapter and *must* be null terminated. @@ -83,7 +83,7 @@ enum uma_result_t umaMemoryProviderFree(uma_memory_provider_handle_t hProvider, /// \param ppMessage [out] pointer to a string containing provider specific /// result in string representation /// \param pError [out] pointer to an integer where the adapter specific error code will be stored -void umaMemoryProviderGetLastNativeError(uma_memory_provider_handle_t hProvider, +void umfMemoryProviderGetLastNativeError(umf_memory_provider_handle_t hProvider, const char **ppMessage, int32_t *pError); @@ -92,9 +92,9 @@ void umaMemoryProviderGetLastNativeError(uma_memory_provider_handle_t hProvider, /// \param hProvider handle to the memory provider /// \param size allocation size /// \param pageSize [out] will be updated with recommended page size -/// \return UMA_RESULT_SUCCESS on success or appropriate error code on failure. -enum uma_result_t -umaMemoryProviderGetRecommendedPageSize(uma_memory_provider_handle_t hProvider, +/// \return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +enum umf_result_t +umfMemoryProviderGetRecommendedPageSize(umf_memory_provider_handle_t hProvider, size_t size, size_t *pageSize); /// @@ -103,9 +103,9 @@ umaMemoryProviderGetRecommendedPageSize(uma_memory_provider_handle_t hProvider, /// \param hProvider handle to the memory provider /// \param ptr [optional] pointer to memory allocated by this memory provider /// \param pageSize [out] will be updated with page size value. -/// \return UMA_RESULT_SUCCESS on success or appropriate error code on failure. -enum uma_result_t -umaMemoryProviderGetMinPageSize(uma_memory_provider_handle_t hProvider, +/// \return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +enum umf_result_t +umfMemoryProviderGetMinPageSize(umf_memory_provider_handle_t hProvider, void *ptr, size_t *pageSize); /// @@ -114,11 +114,11 @@ umaMemoryProviderGetMinPageSize(uma_memory_provider_handle_t hProvider, /// \param hProvider handle to the memory provider /// \param ptr beginning of the virtual memory range /// \param size size of the virtual memory range -/// \return UMA_RESULT_SUCCESS on success or appropriate error code on failure. -/// UMA_RESULT_ERROR_INVALID_ALIGNMENT if ptr or size is not page-aligned. -/// UMA_RESULT_ERROR_NOT_SUPPORTED if operation is not supported by this provider. -enum uma_result_t -umaMemoryProviderPurgeLazy(uma_memory_provider_handle_t hProvider, void *ptr, +/// \return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +/// UMF_RESULT_ERROR_INVALID_ALIGNMENT if ptr or size is not page-aligned. +/// UMF_RESULT_ERROR_NOT_SUPPORTED if operation is not supported by this provider. +enum umf_result_t +umfMemoryProviderPurgeLazy(umf_memory_provider_handle_t hProvider, void *ptr, size_t size); /// @@ -127,31 +127,31 @@ umaMemoryProviderPurgeLazy(uma_memory_provider_handle_t hProvider, void *ptr, /// \param hProvider handle to the memory provider /// \param ptr beginning of the virtual memory range /// \param size size of the virtual memory range -/// \return UMA_RESULT_SUCCESS on success or appropriate error code on failure -/// UMA_RESULT_ERROR_INVALID_ALIGNMENT if ptr or size is not page-aligned. -/// UMA_RESULT_ERROR_NOT_SUPPORTED if operation is not supported by this provider. -enum uma_result_t -umaMemoryProviderPurgeForce(uma_memory_provider_handle_t hProvider, void *ptr, +/// \return UMF_RESULT_SUCCESS on success or appropriate error code on failure +/// UMF_RESULT_ERROR_INVALID_ALIGNMENT if ptr or size is not page-aligned. +/// UMF_RESULT_ERROR_NOT_SUPPORTED if operation is not supported by this provider. +enum umf_result_t +umfMemoryProviderPurgeForce(umf_memory_provider_handle_t hProvider, void *ptr, size_t size); /// /// \brief Retrive name of a given memory provider. /// \param hProvider handle to the memory provider /// \param ppName [out] pointer to a string containing name of the provider -const char *umaMemoryProviderGetName(uma_memory_provider_handle_t hProvider); +const char *umfMemoryProviderGetName(umf_memory_provider_handle_t hProvider); /// \brief Retrieve handle to the last memory provider that returned status other -/// than UMA_RESULT_SUCCESS on the calling thread. +/// than UMF_RESULT_SUCCESS on the calling thread. /// /// \details Handle to the memory provider is stored in the thread local /// storage. The handle is updated every time a memory provider -/// returns status other than UMA_RESULT_SUCCESS. +/// returns status other than UMF_RESULT_SUCCESS. /// /// \return Handle to the memory provider -uma_memory_provider_handle_t umaGetLastFailedMemoryProvider(); +umf_memory_provider_handle_t umfGetLastFailedMemoryProvider(); #ifdef __cplusplus } #endif -#endif /* UMA_MEMORY_PROVIDER_H */ +#endif /* UMF_MEMORY_PROVIDER_H */ diff --git a/source/common/unified_memory_allocation/include/uma/memory_provider_ops.h b/source/common/unified_malloc_framework/include/umf/memory_provider_ops.h similarity index 60% rename from source/common/unified_memory_allocation/include/uma/memory_provider_ops.h rename to source/common/unified_malloc_framework/include/umf/memory_provider_ops.h index 404f221cb3..74d16a72a8 100644 --- a/source/common/unified_memory_allocation/include/uma/memory_provider_ops.h +++ b/source/common/unified_malloc_framework/include/umf/memory_provider_ops.h @@ -8,29 +8,29 @@ * */ -#ifndef UMA_MEMORY_PROVIDER_OPS_H -#define UMA_MEMORY_PROVIDER_OPS_H 1 +#ifndef UMF_MEMORY_PROVIDER_OPS_H +#define UMF_MEMORY_PROVIDER_OPS_H 1 -#include +#include #ifdef __cplusplus extern "C" { #endif /// This structure comprises function pointers used by corresponding -/// umaMemoryProvider* calls. Each memory provider implementation should +/// umfMemoryProvider* calls. Each memory provider implementation should /// initialize all function pointers. -struct uma_memory_provider_ops_t { +struct umf_memory_provider_ops_t { /// Version of the ops structure. - /// Should be initialized using UMA_VERSION_CURRENT + /// Should be initialized using UMF_VERSION_CURRENT uint32_t version; /// /// \brief Intializes memory pool. /// \param params pool-specific params /// \param pool returns pointer to the pool - /// \return UMA_RESULT_SUCCESS on success or appropriate error code on failure. - enum uma_result_t (*initialize)(void *params, void **pool); + /// \return UMF_RESULT_SUCCESS on success or appropriate error code on failure. + enum umf_result_t (*initialize)(void *params, void **pool); /// /// \brief Finalizes memory pool. @@ -38,17 +38,17 @@ struct uma_memory_provider_ops_t { void (*finalize)(void *pool); /// Refer to memory_provider.h for description of those functions - enum uma_result_t (*alloc)(void *provider, size_t size, size_t alignment, + enum umf_result_t (*alloc)(void *provider, size_t size, size_t alignment, void **ptr); - enum uma_result_t (*free)(void *provider, void *ptr, size_t size); + enum umf_result_t (*free)(void *provider, void *ptr, size_t size); void (*get_last_native_error)(void *provider, const char **ppMessage, int32_t *pError); - enum uma_result_t (*get_recommended_page_size)(void *provider, size_t size, + enum umf_result_t (*get_recommended_page_size)(void *provider, size_t size, size_t *pageSize); - enum uma_result_t (*get_min_page_size)(void *provider, void *ptr, + enum umf_result_t (*get_min_page_size)(void *provider, void *ptr, size_t *pageSize); - enum uma_result_t (*purge_lazy)(void *provider, void *ptr, size_t size); - enum uma_result_t (*purge_force)(void *provider, void *ptr, size_t size); + enum umf_result_t (*purge_lazy)(void *provider, void *ptr, size_t size); + enum umf_result_t (*purge_force)(void *provider, void *ptr, size_t size); const char *(*get_name)(void *provider); }; @@ -56,4 +56,4 @@ struct uma_memory_provider_ops_t { } #endif -#endif /* #ifndef UMA_MEMORY_PROVIDER_OPS_H */ +#endif /* #ifndef UMF_MEMORY_PROVIDER_OPS_H */ diff --git a/source/common/unified_memory_allocation/src/memory_pool.c b/source/common/unified_malloc_framework/src/memory_pool.c similarity index 54% rename from source/common/unified_memory_allocation/src/memory_pool.c rename to source/common/unified_malloc_framework/src/memory_pool.c index 13a86d8a6c..9328b3896a 100644 --- a/source/common/unified_memory_allocation/src/memory_pool.c +++ b/source/common/unified_malloc_framework/src/memory_pool.c @@ -11,53 +11,53 @@ #include "memory_provider_internal.h" #include "memory_tracker.h" -#include -#include +#include +#include #include #include -struct uma_memory_pool_t { +struct umf_memory_pool_t { void *pool_priv; - struct uma_memory_pool_ops_t ops; + struct umf_memory_pool_ops_t ops; // Holds array of memory providers. All providers are wrapped - // by memory tracking providers (owned and released by UMA). - uma_memory_provider_handle_t *providers; + // by memory tracking providers (owned and released by UMF). + umf_memory_provider_handle_t *providers; size_t numProviders; }; static void -destroyMemoryProviderWrappers(uma_memory_provider_handle_t *providers, +destroyMemoryProviderWrappers(umf_memory_provider_handle_t *providers, size_t numProviders) { for (size_t i = 0; i < numProviders; i++) { - umaMemoryProviderDestroy(providers[i]); + umfMemoryProviderDestroy(providers[i]); } free(providers); } -enum uma_result_t umaPoolCreate(struct uma_memory_pool_ops_t *ops, - uma_memory_provider_handle_t *providers, +enum umf_result_t umfPoolCreate(struct umf_memory_pool_ops_t *ops, + umf_memory_provider_handle_t *providers, size_t numProviders, void *params, - uma_memory_pool_handle_t *hPool) { + umf_memory_pool_handle_t *hPool) { if (!numProviders || !providers) { - return UMA_RESULT_ERROR_INVALID_ARGUMENT; + return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - enum uma_result_t ret = UMA_RESULT_SUCCESS; - uma_memory_pool_handle_t pool = malloc(sizeof(struct uma_memory_pool_t)); + enum umf_result_t ret = UMF_RESULT_SUCCESS; + umf_memory_pool_handle_t pool = malloc(sizeof(struct umf_memory_pool_t)); if (!pool) { - return UMA_RESULT_ERROR_OUT_OF_HOST_MEMORY; + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } - assert(ops->version == UMA_VERSION_CURRENT); + assert(ops->version == UMF_VERSION_CURRENT); pool->providers = - calloc(numProviders, sizeof(uma_memory_provider_handle_t)); + calloc(numProviders, sizeof(umf_memory_provider_handle_t)); if (!pool->providers) { - ret = UMA_RESULT_ERROR_OUT_OF_HOST_MEMORY; + ret = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; goto err_providers_alloc; } @@ -66,9 +66,9 @@ enum uma_result_t umaPoolCreate(struct uma_memory_pool_ops_t *ops, // Wrap each provider with memory tracking provider. for (providerInd = 0; providerInd < numProviders; providerInd++) { - ret = umaTrackingMemoryProviderCreate(providers[providerInd], pool, + ret = umfTrackingMemoryProviderCreate(providers[providerInd], pool, &pool->providers[providerInd]); - if (ret != UMA_RESULT_SUCCESS) { + if (ret != UMF_RESULT_SUCCESS) { goto err_providers_init; } } @@ -76,12 +76,12 @@ enum uma_result_t umaPoolCreate(struct uma_memory_pool_ops_t *ops, pool->ops = *ops; ret = ops->initialize(pool->providers, pool->numProviders, params, &pool->pool_priv); - if (ret != UMA_RESULT_SUCCESS) { + if (ret != UMF_RESULT_SUCCESS) { goto err_pool_init; } *hPool = pool; - return UMA_RESULT_SUCCESS; + return UMF_RESULT_SUCCESS; err_pool_init: err_providers_init: @@ -92,59 +92,59 @@ enum uma_result_t umaPoolCreate(struct uma_memory_pool_ops_t *ops, return ret; } -void umaPoolDestroy(uma_memory_pool_handle_t hPool) { +void umfPoolDestroy(umf_memory_pool_handle_t hPool) { hPool->ops.finalize(hPool->pool_priv); destroyMemoryProviderWrappers(hPool->providers, hPool->numProviders); free(hPool); } -void *umaPoolMalloc(uma_memory_pool_handle_t hPool, size_t size) { +void *umfPoolMalloc(umf_memory_pool_handle_t hPool, size_t size) { return hPool->ops.malloc(hPool->pool_priv, size); } -void *umaPoolAlignedMalloc(uma_memory_pool_handle_t hPool, size_t size, +void *umfPoolAlignedMalloc(umf_memory_pool_handle_t hPool, size_t size, size_t alignment) { return hPool->ops.aligned_malloc(hPool->pool_priv, size, alignment); } -void *umaPoolCalloc(uma_memory_pool_handle_t hPool, size_t num, size_t size) { +void *umfPoolCalloc(umf_memory_pool_handle_t hPool, size_t num, size_t size) { return hPool->ops.calloc(hPool->pool_priv, num, size); } -void *umaPoolRealloc(uma_memory_pool_handle_t hPool, void *ptr, size_t size) { +void *umfPoolRealloc(umf_memory_pool_handle_t hPool, void *ptr, size_t size) { return hPool->ops.realloc(hPool->pool_priv, ptr, size); } -size_t umaPoolMallocUsableSize(uma_memory_pool_handle_t hPool, void *ptr) { +size_t umfPoolMallocUsableSize(umf_memory_pool_handle_t hPool, void *ptr) { return hPool->ops.malloc_usable_size(hPool->pool_priv, ptr); } -void umaPoolFree(uma_memory_pool_handle_t hPool, void *ptr) { +void umfPoolFree(umf_memory_pool_handle_t hPool, void *ptr) { hPool->ops.free(hPool->pool_priv, ptr); } -void umaFree(void *ptr) { - uma_memory_pool_handle_t hPool = umaPoolByPtr(ptr); +void umfFree(void *ptr) { + umf_memory_pool_handle_t hPool = umfPoolByPtr(ptr); if (hPool) { - umaPoolFree(hPool, ptr); + umfPoolFree(hPool, ptr); } } -enum uma_result_t -umaPoolGetLastAllocationError(uma_memory_pool_handle_t hPool) { +enum umf_result_t +umfPoolGetLastAllocationError(umf_memory_pool_handle_t hPool) { return hPool->ops.get_last_allocation_error(hPool->pool_priv); } -uma_memory_pool_handle_t umaPoolByPtr(const void *ptr) { - return umaMemoryTrackerGetPool(umaMemoryTrackerGet(), ptr); +umf_memory_pool_handle_t umfPoolByPtr(const void *ptr) { + return umfMemoryTrackerGetPool(umfMemoryTrackerGet(), ptr); } -enum uma_result_t -umaPoolGetMemoryProviders(uma_memory_pool_handle_t hPool, size_t numProviders, - uma_memory_provider_handle_t *hProviders, +enum umf_result_t +umfPoolGetMemoryProviders(umf_memory_pool_handle_t hPool, size_t numProviders, + umf_memory_provider_handle_t *hProviders, size_t *numProvidersRet) { if (hProviders && numProviders < hPool->numProviders) { - return UMA_RESULT_ERROR_INVALID_ARGUMENT; + return UMF_RESULT_ERROR_INVALID_ARGUMENT; } if (numProvidersRet) { @@ -153,10 +153,10 @@ umaPoolGetMemoryProviders(uma_memory_pool_handle_t hPool, size_t numProviders, if (hProviders) { for (size_t i = 0; i < hPool->numProviders; i++) { - umaTrackingMemoryProviderGetUpstreamProvider( - umaMemoryProviderGetPriv(hPool->providers[i]), hProviders + i); + umfTrackingMemoryProviderGetUpstreamProvider( + umfMemoryProviderGetPriv(hPool->providers[i]), hProviders + i); } } - return UMA_RESULT_SUCCESS; + return UMF_RESULT_SUCCESS; } diff --git a/source/common/unified_memory_allocation/src/memory_provider.c b/source/common/unified_malloc_framework/src/memory_provider.c similarity index 56% rename from source/common/unified_memory_allocation/src/memory_provider.c rename to source/common/unified_malloc_framework/src/memory_provider.c index 103bcef88a..38e4dfbf8b 100644 --- a/source/common/unified_memory_allocation/src/memory_provider.c +++ b/source/common/unified_malloc_framework/src/memory_provider.c @@ -9,32 +9,32 @@ */ #include "memory_provider_internal.h" -#include +#include #include #include -struct uma_memory_provider_t { - struct uma_memory_provider_ops_t ops; +struct umf_memory_provider_t { + struct umf_memory_provider_ops_t ops; void *provider_priv; }; -enum uma_result_t -umaMemoryProviderCreate(struct uma_memory_provider_ops_t *ops, void *params, - uma_memory_provider_handle_t *hProvider) { - uma_memory_provider_handle_t provider = - malloc(sizeof(struct uma_memory_provider_t)); +enum umf_result_t +umfMemoryProviderCreate(struct umf_memory_provider_ops_t *ops, void *params, + umf_memory_provider_handle_t *hProvider) { + umf_memory_provider_handle_t provider = + malloc(sizeof(struct umf_memory_provider_t)); if (!provider) { - return UMA_RESULT_ERROR_OUT_OF_HOST_MEMORY; + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } - assert(ops->version == UMA_VERSION_CURRENT); + assert(ops->version == UMF_VERSION_CURRENT); provider->ops = *ops; void *provider_priv; - enum uma_result_t ret = ops->initialize(params, &provider_priv); - if (ret != UMA_RESULT_SUCCESS) { + enum umf_result_t ret = ops->initialize(params, &provider_priv); + if (ret != UMF_RESULT_SUCCESS) { free(provider); return ret; } @@ -43,90 +43,90 @@ umaMemoryProviderCreate(struct uma_memory_provider_ops_t *ops, void *params, *hProvider = provider; - return UMA_RESULT_SUCCESS; + return UMF_RESULT_SUCCESS; } -void umaMemoryProviderDestroy(uma_memory_provider_handle_t hProvider) { +void umfMemoryProviderDestroy(umf_memory_provider_handle_t hProvider) { hProvider->ops.finalize(hProvider->provider_priv); free(hProvider); } static void -checkErrorAndSetLastProvider(enum uma_result_t result, - uma_memory_provider_handle_t hProvider) { - if (result != UMA_RESULT_SUCCESS) { - *umaGetLastFailedMemoryProviderPtr() = hProvider; +checkErrorAndSetLastProvider(enum umf_result_t result, + umf_memory_provider_handle_t hProvider) { + if (result != UMF_RESULT_SUCCESS) { + *umfGetLastFailedMemoryProviderPtr() = hProvider; } } -enum uma_result_t umaMemoryProviderAlloc(uma_memory_provider_handle_t hProvider, +enum umf_result_t umfMemoryProviderAlloc(umf_memory_provider_handle_t hProvider, size_t size, size_t alignment, void **ptr) { - enum uma_result_t res = + enum umf_result_t res = hProvider->ops.alloc(hProvider->provider_priv, size, alignment, ptr); checkErrorAndSetLastProvider(res, hProvider); return res; } -enum uma_result_t umaMemoryProviderFree(uma_memory_provider_handle_t hProvider, +enum umf_result_t umfMemoryProviderFree(umf_memory_provider_handle_t hProvider, void *ptr, size_t size) { - enum uma_result_t res = + enum umf_result_t res = hProvider->ops.free(hProvider->provider_priv, ptr, size); checkErrorAndSetLastProvider(res, hProvider); return res; } -void umaMemoryProviderGetLastNativeError(uma_memory_provider_handle_t hProvider, +void umfMemoryProviderGetLastNativeError(umf_memory_provider_handle_t hProvider, const char **ppMessage, int32_t *pError) { hProvider->ops.get_last_native_error(hProvider->provider_priv, ppMessage, pError); } -void *umaMemoryProviderGetPriv(uma_memory_provider_handle_t hProvider) { +void *umfMemoryProviderGetPriv(umf_memory_provider_handle_t hProvider) { return hProvider->provider_priv; } -enum uma_result_t -umaMemoryProviderGetRecommendedPageSize(uma_memory_provider_handle_t hProvider, +enum umf_result_t +umfMemoryProviderGetRecommendedPageSize(umf_memory_provider_handle_t hProvider, size_t size, size_t *pageSize) { - enum uma_result_t res = hProvider->ops.get_recommended_page_size( + enum umf_result_t res = hProvider->ops.get_recommended_page_size( hProvider->provider_priv, size, pageSize); checkErrorAndSetLastProvider(res, hProvider); return res; } -enum uma_result_t -umaMemoryProviderGetMinPageSize(uma_memory_provider_handle_t hProvider, +enum umf_result_t +umfMemoryProviderGetMinPageSize(umf_memory_provider_handle_t hProvider, void *ptr, size_t *pageSize) { - enum uma_result_t res = hProvider->ops.get_min_page_size( + enum umf_result_t res = hProvider->ops.get_min_page_size( hProvider->provider_priv, ptr, pageSize); checkErrorAndSetLastProvider(res, hProvider); return res; } -enum uma_result_t -umaMemoryProviderPurgeLazy(uma_memory_provider_handle_t hProvider, void *ptr, +enum umf_result_t +umfMemoryProviderPurgeLazy(umf_memory_provider_handle_t hProvider, void *ptr, size_t size) { - enum uma_result_t res = + enum umf_result_t res = hProvider->ops.purge_lazy(hProvider->provider_priv, ptr, size); checkErrorAndSetLastProvider(res, hProvider); return res; } -enum uma_result_t -umaMemoryProviderPurgeForce(uma_memory_provider_handle_t hProvider, void *ptr, +enum umf_result_t +umfMemoryProviderPurgeForce(umf_memory_provider_handle_t hProvider, void *ptr, size_t size) { - enum uma_result_t res = + enum umf_result_t res = hProvider->ops.purge_force(hProvider->provider_priv, ptr, size); checkErrorAndSetLastProvider(res, hProvider); return res; } -const char *umaMemoryProviderGetName(uma_memory_provider_handle_t hProvider) { +const char *umfMemoryProviderGetName(umf_memory_provider_handle_t hProvider) { return hProvider->ops.get_name(hProvider->provider_priv); } -uma_memory_provider_handle_t umaGetLastFailedMemoryProvider() { - return *umaGetLastFailedMemoryProviderPtr(); +umf_memory_provider_handle_t umfGetLastFailedMemoryProvider() { + return *umfGetLastFailedMemoryProviderPtr(); } diff --git a/source/common/unified_memory_allocation/src/memory_provider_get_last_failed.cpp b/source/common/unified_malloc_framework/src/memory_provider_get_last_failed.cpp similarity index 72% rename from source/common/unified_memory_allocation/src/memory_provider_get_last_failed.cpp rename to source/common/unified_malloc_framework/src/memory_provider_get_last_failed.cpp index 52a4e6ce35..c439213a26 100644 --- a/source/common/unified_memory_allocation/src/memory_provider_get_last_failed.cpp +++ b/source/common/unified_malloc_framework/src/memory_provider_get_last_failed.cpp @@ -12,9 +12,9 @@ extern "C" { -static thread_local uma_memory_provider_handle_t lastFailedProvider = nullptr; +static thread_local umf_memory_provider_handle_t lastFailedProvider = nullptr; -uma_memory_provider_handle_t *umaGetLastFailedMemoryProviderPtr() { +umf_memory_provider_handle_t *umfGetLastFailedMemoryProviderPtr() { return &lastFailedProvider; } } diff --git a/source/common/unified_memory_allocation/src/memory_provider_internal.h b/source/common/unified_malloc_framework/src/memory_provider_internal.h similarity index 52% rename from source/common/unified_memory_allocation/src/memory_provider_internal.h rename to source/common/unified_malloc_framework/src/memory_provider_internal.h index a2b190488f..07befd4b4e 100644 --- a/source/common/unified_memory_allocation/src/memory_provider_internal.h +++ b/source/common/unified_malloc_framework/src/memory_provider_internal.h @@ -8,20 +8,20 @@ * */ -#ifndef UMA_MEMORY_PROVIDER_INTERNAL_H -#define UMA_MEMORY_PROVIDER_INTERNAL_H 1 +#ifndef UMF_MEMORY_PROVIDER_INTERNAL_H +#define UMF_MEMORY_PROVIDER_INTERNAL_H 1 -#include +#include #ifdef __cplusplus extern "C" { #endif -void *umaMemoryProviderGetPriv(uma_memory_provider_handle_t hProvider); -uma_memory_provider_handle_t *umaGetLastFailedMemoryProviderPtr(); +void *umfMemoryProviderGetPriv(umf_memory_provider_handle_t hProvider); +umf_memory_provider_handle_t *umfGetLastFailedMemoryProviderPtr(); #ifdef __cplusplus } #endif -#endif /* UMA_MEMORY_PROVIDER_INTERNAL_H */ +#endif /* UMF_MEMORY_PROVIDER_INTERNAL_H */ diff --git a/source/common/unified_malloc_framework/src/memory_tracker.cpp b/source/common/unified_malloc_framework/src/memory_tracker.cpp new file mode 100644 index 0000000000..74638579cc --- /dev/null +++ b/source/common/unified_malloc_framework/src/memory_tracker.cpp @@ -0,0 +1,247 @@ +/* + * + * Copyright (C) 2023 Intel Corporation + * + * Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. + * See LICENSE.TXT + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#include "memory_tracker.h" +#include +#include + +#include +#include +#include +#include +#include + +// TODO: reimplement in C and optimize... +struct umf_memory_tracker_t { + enum umf_result_t add(void *pool, const void *ptr, size_t size) { + std::unique_lock lock(mtx); + + if (size == 0) { + return UMF_RESULT_SUCCESS; + } + + auto ret = + map.try_emplace(reinterpret_cast(ptr), size, pool); + return ret.second ? UMF_RESULT_SUCCESS : UMF_RESULT_ERROR_UNKNOWN; + } + + enum umf_result_t remove(const void *ptr, size_t size) { + std::unique_lock lock(mtx); + + map.erase(reinterpret_cast(ptr)); + + // TODO: handle removing part of the range + (void)size; + + return UMF_RESULT_SUCCESS; + } + + void *find(const void *ptr) { + std::shared_lock lock(mtx); + + auto intptr = reinterpret_cast(ptr); + auto it = map.upper_bound(intptr); + if (it == map.begin()) { + return nullptr; + } + + --it; + + auto address = it->first; + auto size = it->second.first; + auto pool = it->second.second; + + if (intptr >= address && intptr < address + size) { + return pool; + } + + return nullptr; + } + + private: + std::shared_mutex mtx; + std::map> map; +}; + +static enum umf_result_t +umfMemoryTrackerAdd(umf_memory_tracker_handle_t hTracker, void *pool, + const void *ptr, size_t size) { + return hTracker->add(pool, ptr, size); +} + +static enum umf_result_t +umfMemoryTrackerRemove(umf_memory_tracker_handle_t hTracker, const void *ptr, + size_t size) { + return hTracker->remove(ptr, size); +} + +extern "C" { + +umf_memory_tracker_handle_t umfMemoryTrackerGet(void) { + static umf_memory_tracker_t tracker; + return &tracker; +} + +void *umfMemoryTrackerGetPool(umf_memory_tracker_handle_t hTracker, + const void *ptr) { + return hTracker->find(ptr); +} + +struct umf_tracking_memory_provider_t { + umf_memory_provider_handle_t hUpstream; + umf_memory_tracker_handle_t hTracker; + umf_memory_pool_handle_t pool; +}; + +typedef struct umf_tracking_memory_provider_t umf_tracking_memory_provider_t; + +static enum umf_result_t trackingAlloc(void *hProvider, size_t size, + size_t alignment, void **ptr) { + umf_tracking_memory_provider_t *p = + (umf_tracking_memory_provider_t *)hProvider; + enum umf_result_t ret = UMF_RESULT_SUCCESS; + + if (!p->hUpstream) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + ret = umfMemoryProviderAlloc(p->hUpstream, size, alignment, ptr); + if (ret != UMF_RESULT_SUCCESS) { + return ret; + } + + ret = umfMemoryTrackerAdd(p->hTracker, p->pool, *ptr, size); + if (ret != UMF_RESULT_SUCCESS && p->hUpstream) { + if (umfMemoryProviderFree(p->hUpstream, *ptr, size)) { + // TODO: LOG + } + } + + return ret; +} + +static enum umf_result_t trackingFree(void *hProvider, void *ptr, size_t size) { + enum umf_result_t ret; + umf_tracking_memory_provider_t *p = + (umf_tracking_memory_provider_t *)hProvider; + + // umfMemoryTrackerRemove should be called before umfMemoryProviderFree + // to avoid a race condition. If the order would be different, other thread + // could allocate the memory at address `ptr` before a call to umfMemoryTrackerRemove + // resulting in inconsistent state. + ret = umfMemoryTrackerRemove(p->hTracker, ptr, size); + if (ret != UMF_RESULT_SUCCESS) { + return ret; + } + + ret = umfMemoryProviderFree(p->hUpstream, ptr, size); + if (ret != UMF_RESULT_SUCCESS) { + if (umfMemoryTrackerAdd(p->hTracker, p->pool, ptr, size) != + UMF_RESULT_SUCCESS) { + // TODO: LOG + } + return ret; + } + + return ret; +} + +static enum umf_result_t trackingInitialize(void *params, void **ret) { + umf_tracking_memory_provider_t *provider = + (umf_tracking_memory_provider_t *)malloc( + sizeof(umf_tracking_memory_provider_t)); + if (!provider) { + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + *provider = *((umf_tracking_memory_provider_t *)params); + *ret = provider; + return UMF_RESULT_SUCCESS; +} + +static void trackingFinalize(void *provider) { free(provider); } + +static void trackingGetLastError(void *provider, const char **msg, + int32_t *pError) { + umf_tracking_memory_provider_t *p = + (umf_tracking_memory_provider_t *)provider; + umfMemoryProviderGetLastNativeError(p->hUpstream, msg, pError); +} + +static enum umf_result_t +trackingGetRecommendedPageSize(void *provider, size_t size, size_t *pageSize) { + umf_tracking_memory_provider_t *p = + (umf_tracking_memory_provider_t *)provider; + return umfMemoryProviderGetRecommendedPageSize(p->hUpstream, size, + pageSize); +} + +static enum umf_result_t trackingGetMinPageSize(void *provider, void *ptr, + size_t *pageSize) { + umf_tracking_memory_provider_t *p = + (umf_tracking_memory_provider_t *)provider; + return umfMemoryProviderGetMinPageSize(p->hUpstream, ptr, pageSize); +} + +static enum umf_result_t trackingPurgeLazy(void *provider, void *ptr, + size_t size) { + umf_tracking_memory_provider_t *p = + (umf_tracking_memory_provider_t *)provider; + return umfMemoryProviderPurgeLazy(p->hUpstream, ptr, size); +} + +static enum umf_result_t trackingPurgeForce(void *provider, void *ptr, + size_t size) { + umf_tracking_memory_provider_t *p = + (umf_tracking_memory_provider_t *)provider; + return umfMemoryProviderPurgeForce(p->hUpstream, ptr, size); +} + +static const char *trackingName(void *provider) { + umf_tracking_memory_provider_t *p = + (umf_tracking_memory_provider_t *)provider; + return umfMemoryProviderGetName(p->hUpstream); +} + +enum umf_result_t umfTrackingMemoryProviderCreate( + umf_memory_provider_handle_t hUpstream, umf_memory_pool_handle_t hPool, + umf_memory_provider_handle_t *hTrackingProvider) { + umf_tracking_memory_provider_t params; + params.hUpstream = hUpstream; + params.hTracker = umfMemoryTrackerGet(); + params.pool = hPool; + + struct umf_memory_provider_ops_t trackingMemoryProviderOps; + trackingMemoryProviderOps.version = UMF_VERSION_CURRENT; + trackingMemoryProviderOps.initialize = trackingInitialize; + trackingMemoryProviderOps.finalize = trackingFinalize; + trackingMemoryProviderOps.alloc = trackingAlloc; + trackingMemoryProviderOps.free = trackingFree; + trackingMemoryProviderOps.get_last_native_error = trackingGetLastError; + trackingMemoryProviderOps.get_min_page_size = trackingGetMinPageSize; + trackingMemoryProviderOps.get_recommended_page_size = + trackingGetRecommendedPageSize; + trackingMemoryProviderOps.purge_force = trackingPurgeForce; + trackingMemoryProviderOps.purge_lazy = trackingPurgeLazy; + trackingMemoryProviderOps.get_name = trackingName; + + return umfMemoryProviderCreate(&trackingMemoryProviderOps, ¶ms, + hTrackingProvider); +} + +void umfTrackingMemoryProviderGetUpstreamProvider( + umf_memory_provider_handle_t hTrackingProvider, + umf_memory_provider_handle_t *hUpstream) { + assert(hUpstream); + umf_tracking_memory_provider_t *p = + (umf_tracking_memory_provider_t *)hTrackingProvider; + *hUpstream = p->hUpstream; +} +} diff --git a/source/common/unified_malloc_framework/src/memory_tracker.h b/source/common/unified_malloc_framework/src/memory_tracker.h new file mode 100644 index 0000000000..c6e620bf66 --- /dev/null +++ b/source/common/unified_malloc_framework/src/memory_tracker.h @@ -0,0 +1,42 @@ +/* + * + * Copyright (C) 2023 Intel Corporation + * + * Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. + * See LICENSE.TXT + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#ifndef UMF_MEMORY_TRACKER_INTERNAL_H +#define UMF_MEMORY_TRACKER_INTERNAL_H 1 + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct umf_memory_tracker_t *umf_memory_tracker_handle_t; + +umf_memory_tracker_handle_t umfMemoryTrackerGet(void); +void *umfMemoryTrackerGetPool(umf_memory_tracker_handle_t hTracker, + const void *ptr); + +// Creates a memory provider that tracks each allocation/deallocation through umf_memory_tracker_handle_t and +// forwards all requests to hUpstream memory Provider. hUpstream liftime should be managed by the user of this function. +enum umf_result_t umfTrackingMemoryProviderCreate( + umf_memory_provider_handle_t hUpstream, umf_memory_pool_handle_t hPool, + umf_memory_provider_handle_t *hTrackingProvider); + +void umfTrackingMemoryProviderGetUpstreamProvider( + umf_memory_provider_handle_t hTrackingProvider, + umf_memory_provider_handle_t *hUpstream); + +#ifdef __cplusplus +} +#endif + +#endif /* UMF_MEMORY_TRACKER_INTERNAL_H */ diff --git a/source/common/unified_memory_allocation/include/uma/base.h b/source/common/unified_memory_allocation/include/uma/base.h deleted file mode 100644 index bc00e2db3a..0000000000 --- a/source/common/unified_memory_allocation/include/uma/base.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * - * Copyright (C) 2023 Intel Corporation - * - * Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. - * See LICENSE.TXT - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - * - */ - -#ifndef UMA_BASE_H -#define UMA_BASE_H 1 - -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/// \brief Generates generic 'UMA' API versions -#define UMA_MAKE_VERSION(_major, _minor) \ - ((_major << 16) | (_minor & 0x0000ffff)) - -/// \brief Extracts 'UMA' API major version -#define UMA_MAJOR_VERSION(_ver) (_ver >> 16) - -/// \brief Extracts 'UMA' API minor version -#define UMA_MINOR_VERSION(_ver) (_ver & 0x0000ffff) - -/// \brief Current version of the UMA headers -#define UMA_VERSION_CURRENT UMA_MAKE_VERSION(0, 9) - -/// \brief Operation results -enum uma_result_t { - UMA_RESULT_SUCCESS = 0, ///< Success - UMA_RESULT_ERROR_OUT_OF_HOST_MEMORY = - 1, ///< Insufficient host memory to satisfy call, - UMA_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC = - 2, ///< A provider specific warning/error has been reported and can be - ///< Retrieved via the umaMemoryProviderGetLastNativeError entry point. - UMA_RESULT_ERROR_INVALID_ARGUMENT = - 3, ///< Generic error code for invalid arguments - UMA_RESULT_ERROR_INVALID_ALIGNMENT = 4, /// Invalid alignment of an argument - UMA_RESULT_ERROR_NOT_SUPPORTED = 5, /// Operation not supported - - UMA_RESULT_ERROR_UNKNOWN = 0x7ffffffe ///< Unknown or internal error -}; - -#ifdef __cplusplus -} -#endif - -#endif /* UMA_BASE_H */ diff --git a/source/common/unified_memory_allocation/src/memory_tracker.cpp b/source/common/unified_memory_allocation/src/memory_tracker.cpp deleted file mode 100644 index 32f7dff0b1..0000000000 --- a/source/common/unified_memory_allocation/src/memory_tracker.cpp +++ /dev/null @@ -1,247 +0,0 @@ -/* - * - * Copyright (C) 2023 Intel Corporation - * - * Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. - * See LICENSE.TXT - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - * - */ - -#include "memory_tracker.h" -#include -#include - -#include -#include -#include -#include -#include - -// TODO: reimplement in C and optimize... -struct uma_memory_tracker_t { - enum uma_result_t add(void *pool, const void *ptr, size_t size) { - std::unique_lock lock(mtx); - - if (size == 0) { - return UMA_RESULT_SUCCESS; - } - - auto ret = - map.try_emplace(reinterpret_cast(ptr), size, pool); - return ret.second ? UMA_RESULT_SUCCESS : UMA_RESULT_ERROR_UNKNOWN; - } - - enum uma_result_t remove(const void *ptr, size_t size) { - std::unique_lock lock(mtx); - - map.erase(reinterpret_cast(ptr)); - - // TODO: handle removing part of the range - (void)size; - - return UMA_RESULT_SUCCESS; - } - - void *find(const void *ptr) { - std::shared_lock lock(mtx); - - auto intptr = reinterpret_cast(ptr); - auto it = map.upper_bound(intptr); - if (it == map.begin()) { - return nullptr; - } - - --it; - - auto address = it->first; - auto size = it->second.first; - auto pool = it->second.second; - - if (intptr >= address && intptr < address + size) { - return pool; - } - - return nullptr; - } - - private: - std::shared_mutex mtx; - std::map> map; -}; - -static enum uma_result_t -umaMemoryTrackerAdd(uma_memory_tracker_handle_t hTracker, void *pool, - const void *ptr, size_t size) { - return hTracker->add(pool, ptr, size); -} - -static enum uma_result_t -umaMemoryTrackerRemove(uma_memory_tracker_handle_t hTracker, const void *ptr, - size_t size) { - return hTracker->remove(ptr, size); -} - -extern "C" { - -uma_memory_tracker_handle_t umaMemoryTrackerGet(void) { - static uma_memory_tracker_t tracker; - return &tracker; -} - -void *umaMemoryTrackerGetPool(uma_memory_tracker_handle_t hTracker, - const void *ptr) { - return hTracker->find(ptr); -} - -struct uma_tracking_memory_provider_t { - uma_memory_provider_handle_t hUpstream; - uma_memory_tracker_handle_t hTracker; - uma_memory_pool_handle_t pool; -}; - -typedef struct uma_tracking_memory_provider_t uma_tracking_memory_provider_t; - -static enum uma_result_t trackingAlloc(void *hProvider, size_t size, - size_t alignment, void **ptr) { - uma_tracking_memory_provider_t *p = - (uma_tracking_memory_provider_t *)hProvider; - enum uma_result_t ret = UMA_RESULT_SUCCESS; - - if (!p->hUpstream) { - return UMA_RESULT_ERROR_INVALID_ARGUMENT; - } - - ret = umaMemoryProviderAlloc(p->hUpstream, size, alignment, ptr); - if (ret != UMA_RESULT_SUCCESS) { - return ret; - } - - ret = umaMemoryTrackerAdd(p->hTracker, p->pool, *ptr, size); - if (ret != UMA_RESULT_SUCCESS && p->hUpstream) { - if (umaMemoryProviderFree(p->hUpstream, *ptr, size)) { - // TODO: LOG - } - } - - return ret; -} - -static enum uma_result_t trackingFree(void *hProvider, void *ptr, size_t size) { - enum uma_result_t ret; - uma_tracking_memory_provider_t *p = - (uma_tracking_memory_provider_t *)hProvider; - - // umaMemoryTrackerRemove should be called before umaMemoryProviderFree - // to avoid a race condition. If the order would be different, other thread - // could allocate the memory at address `ptr` before a call to umaMemoryTrackerRemove - // resulting in inconsistent state. - ret = umaMemoryTrackerRemove(p->hTracker, ptr, size); - if (ret != UMA_RESULT_SUCCESS) { - return ret; - } - - ret = umaMemoryProviderFree(p->hUpstream, ptr, size); - if (ret != UMA_RESULT_SUCCESS) { - if (umaMemoryTrackerAdd(p->hTracker, p->pool, ptr, size) != - UMA_RESULT_SUCCESS) { - // TODO: LOG - } - return ret; - } - - return ret; -} - -static enum uma_result_t trackingInitialize(void *params, void **ret) { - uma_tracking_memory_provider_t *provider = - (uma_tracking_memory_provider_t *)malloc( - sizeof(uma_tracking_memory_provider_t)); - if (!provider) { - return UMA_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - *provider = *((uma_tracking_memory_provider_t *)params); - *ret = provider; - return UMA_RESULT_SUCCESS; -} - -static void trackingFinalize(void *provider) { free(provider); } - -static void trackingGetLastError(void *provider, const char **msg, - int32_t *pError) { - uma_tracking_memory_provider_t *p = - (uma_tracking_memory_provider_t *)provider; - umaMemoryProviderGetLastNativeError(p->hUpstream, msg, pError); -} - -static enum uma_result_t -trackingGetRecommendedPageSize(void *provider, size_t size, size_t *pageSize) { - uma_tracking_memory_provider_t *p = - (uma_tracking_memory_provider_t *)provider; - return umaMemoryProviderGetRecommendedPageSize(p->hUpstream, size, - pageSize); -} - -static enum uma_result_t trackingGetMinPageSize(void *provider, void *ptr, - size_t *pageSize) { - uma_tracking_memory_provider_t *p = - (uma_tracking_memory_provider_t *)provider; - return umaMemoryProviderGetMinPageSize(p->hUpstream, ptr, pageSize); -} - -static enum uma_result_t trackingPurgeLazy(void *provider, void *ptr, - size_t size) { - uma_tracking_memory_provider_t *p = - (uma_tracking_memory_provider_t *)provider; - return umaMemoryProviderPurgeLazy(p->hUpstream, ptr, size); -} - -static enum uma_result_t trackingPurgeForce(void *provider, void *ptr, - size_t size) { - uma_tracking_memory_provider_t *p = - (uma_tracking_memory_provider_t *)provider; - return umaMemoryProviderPurgeForce(p->hUpstream, ptr, size); -} - -static const char *trackingName(void *provider) { - uma_tracking_memory_provider_t *p = - (uma_tracking_memory_provider_t *)provider; - return umaMemoryProviderGetName(p->hUpstream); -} - -enum uma_result_t umaTrackingMemoryProviderCreate( - uma_memory_provider_handle_t hUpstream, uma_memory_pool_handle_t hPool, - uma_memory_provider_handle_t *hTrackingProvider) { - uma_tracking_memory_provider_t params; - params.hUpstream = hUpstream; - params.hTracker = umaMemoryTrackerGet(); - params.pool = hPool; - - struct uma_memory_provider_ops_t trackingMemoryProviderOps; - trackingMemoryProviderOps.version = UMA_VERSION_CURRENT; - trackingMemoryProviderOps.initialize = trackingInitialize; - trackingMemoryProviderOps.finalize = trackingFinalize; - trackingMemoryProviderOps.alloc = trackingAlloc; - trackingMemoryProviderOps.free = trackingFree; - trackingMemoryProviderOps.get_last_native_error = trackingGetLastError; - trackingMemoryProviderOps.get_min_page_size = trackingGetMinPageSize; - trackingMemoryProviderOps.get_recommended_page_size = - trackingGetRecommendedPageSize; - trackingMemoryProviderOps.purge_force = trackingPurgeForce; - trackingMemoryProviderOps.purge_lazy = trackingPurgeLazy; - trackingMemoryProviderOps.get_name = trackingName; - - return umaMemoryProviderCreate(&trackingMemoryProviderOps, ¶ms, - hTrackingProvider); -} - -void umaTrackingMemoryProviderGetUpstreamProvider( - uma_memory_provider_handle_t hTrackingProvider, - uma_memory_provider_handle_t *hUpstream) { - assert(hUpstream); - uma_tracking_memory_provider_t *p = - (uma_tracking_memory_provider_t *)hTrackingProvider; - *hUpstream = p->hUpstream; -} -} diff --git a/source/common/unified_memory_allocation/src/memory_tracker.h b/source/common/unified_memory_allocation/src/memory_tracker.h deleted file mode 100644 index 055942174f..0000000000 --- a/source/common/unified_memory_allocation/src/memory_tracker.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * - * Copyright (C) 2023 Intel Corporation - * - * Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. - * See LICENSE.TXT - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - * - */ - -#ifndef UMA_MEMORY_TRACKER_INTERNAL_H -#define UMA_MEMORY_TRACKER_INTERNAL_H 1 - -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct uma_memory_tracker_t *uma_memory_tracker_handle_t; - -uma_memory_tracker_handle_t umaMemoryTrackerGet(void); -void *umaMemoryTrackerGetPool(uma_memory_tracker_handle_t hTracker, - const void *ptr); - -// Creates a memory provider that tracks each allocation/deallocation through uma_memory_tracker_handle_t and -// forwards all requests to hUpstream memory Provider. hUpstream liftime should be managed by the user of this function. -enum uma_result_t umaTrackingMemoryProviderCreate( - uma_memory_provider_handle_t hUpstream, uma_memory_pool_handle_t hPool, - uma_memory_provider_handle_t *hTrackingProvider); - -void umaTrackingMemoryProviderGetUpstreamProvider( - uma_memory_provider_handle_t hTrackingProvider, - uma_memory_provider_handle_t *hUpstream); - -#ifdef __cplusplus -} -#endif - -#endif /* UMA_MEMORY_TRACKER_INTERNAL_H */ diff --git a/source/common/ur_pool_manager.hpp b/source/common/ur_pool_manager.hpp index c4cb169fbd..c4da5d149f 100644 --- a/source/common/ur_pool_manager.hpp +++ b/source/common/ur_pool_manager.hpp @@ -128,7 +128,7 @@ inline bool pool_descriptor::equal(const pool_descriptor &lhs, // We want to share a memory pool for sub-devices and sub-sub devices. // Sub-devices and sub-sub-devices might be represented by different ur_device_handle_t but - // they share the same native_handle_t (which is used by UMA provider). + // they share the same native_handle_t (which is used by UMF provider). // Ref: https://github.com/intel/llvm/commit/86511c5dc84b5781dcfd828caadcb5cac157eae1 // TODO: is this L0 specific? auto ret = urDeviceGetNativeHandle(lhs.hDevice, &lhsNative); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 79265fe742..79ca48236c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -17,7 +17,7 @@ enable_testing() add_subdirectory(python) add_subdirectory(loader) add_subdirectory(conformance) -add_subdirectory(unified_memory_allocation) +add_subdirectory(unified_malloc_framework) add_subdirectory(usm) add_subdirectory(layers) add_subdirectory(unit) diff --git a/test/unified_malloc_framework/CMakeLists.txt b/test/unified_malloc_framework/CMakeLists.txt new file mode 100644 index 0000000000..7636c5d26f --- /dev/null +++ b/test/unified_malloc_framework/CMakeLists.txt @@ -0,0 +1,37 @@ +# Copyright (C) 2022-2023 Intel Corporation +# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +# See LICENSE.TXT +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set(UR_UMF_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + +add_library(umf_test_helpers STATIC + ${UR_UMF_TEST_DIR}/common/pool.c + ${UR_UMF_TEST_DIR}/common/provider.c +) + +target_link_libraries(umf_test_helpers PRIVATE ${PROJECT_NAME}::common) + +function(add_umf_test name) + set(TEST_TARGET_NAME umf_test-${name}) + add_executable(${TEST_TARGET_NAME} + ${ARGN}) + target_link_libraries(${TEST_TARGET_NAME} + PRIVATE + ${PROJECT_NAME}::unified_malloc_framework + ${PROJECT_NAME}::common + umf_test_helpers + GTest::gtest_main) + target_include_directories(${TEST_TARGET_NAME} PRIVATE + ${UR_UMF_TEST_DIR}/common) + add_test(NAME umf-${name} + COMMAND ${TEST_TARGET_NAME} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + set_tests_properties(umf-${name} PROPERTIES LABELS "umf") +endfunction() + +add_subdirectory(umf_pools) + +add_umf_test(memoryProvider memoryProviderAPI.cpp) +add_umf_test(memoryPool memoryPoolAPI.cpp) +add_umf_test(base base.cpp) diff --git a/test/unified_memory_allocation/base.cpp b/test/unified_malloc_framework/base.cpp similarity index 60% rename from test/unified_memory_allocation/base.cpp rename to test/unified_malloc_framework/base.cpp index 49ff6b425d..c9f6211f66 100644 --- a/test/unified_memory_allocation/base.cpp +++ b/test/unified_malloc_framework/base.cpp @@ -5,12 +5,12 @@ #include "base.hpp" -#include +#include -using uma_test::test; +using umf_test::test; TEST_F(test, versionEncodeDecode) { - auto encoded = UMA_MAKE_VERSION(0, 9); - ASSERT_EQ(UMA_MAJOR_VERSION(encoded), 0); - ASSERT_EQ(UMA_MINOR_VERSION(encoded), 9); + auto encoded = UMF_MAKE_VERSION(0, 9); + ASSERT_EQ(UMF_MAJOR_VERSION(encoded), 0); + ASSERT_EQ(UMF_MINOR_VERSION(encoded), 9); } diff --git a/test/unified_memory_allocation/common/base.hpp b/test/unified_malloc_framework/common/base.hpp similarity index 90% rename from test/unified_memory_allocation/common/base.hpp rename to test/unified_malloc_framework/common/base.hpp index d454b3822d..c4d6ccd129 100644 --- a/test/unified_memory_allocation/common/base.hpp +++ b/test/unified_malloc_framework/common/base.hpp @@ -8,12 +8,12 @@ * */ -#ifndef UMA_TEST_BASE_HPP -#define UMA_TEST_BASE_HPP 1 +#ifndef UMF_TEST_BASE_HPP +#define UMF_TEST_BASE_HPP 1 #include -namespace uma_test { +namespace umf_test { #define NOEXCEPT_COND(cond, val, expected_val) \ try { \ @@ -32,6 +32,6 @@ struct test : ::testing::Test { void SetUp() override { ::testing::Test::SetUp(); } void TearDown() override { ::testing::Test::TearDown(); } }; -} // namespace uma_test +} // namespace umf_test -#endif /* UMA_TEST_BASE_HPP */ +#endif /* UMF_TEST_BASE_HPP */ diff --git a/test/unified_memory_allocation/common/pool.c b/test/unified_malloc_framework/common/pool.c similarity index 71% rename from test/unified_memory_allocation/common/pool.c rename to test/unified_malloc_framework/common/pool.c index 200ed75dff..63c8a0c95f 100644 --- a/test/unified_memory_allocation/common/pool.c +++ b/test/unified_malloc_framework/common/pool.c @@ -6,12 +6,12 @@ #include "pool.h" #include "provider.h" -#include +#include #include #include -static enum uma_result_t nullInitialize(uma_memory_provider_handle_t *providers, +static enum umf_result_t nullInitialize(umf_memory_provider_handle_t *providers, size_t numProviders, void *params, void **pool) { (void)providers; @@ -19,7 +19,7 @@ static enum uma_result_t nullInitialize(uma_memory_provider_handle_t *providers, (void)params; assert(providers && numProviders); *pool = NULL; - return UMA_RESULT_SUCCESS; + return UMF_RESULT_SUCCESS; } static void nullFinalize(void *pool) { (void)pool; } @@ -62,14 +62,14 @@ static void nullFree(void *pool, void *ptr) { (void)ptr; } -enum uma_result_t nullGetLastStatus(void *pool) { +enum umf_result_t nullGetLastStatus(void *pool) { (void)pool; - return UMA_RESULT_SUCCESS; + return UMF_RESULT_SUCCESS; } -uma_memory_pool_handle_t nullPoolCreate(void) { - struct uma_memory_pool_ops_t ops = { - .version = UMA_VERSION_CURRENT, +umf_memory_pool_handle_t nullPoolCreate(void) { + struct umf_memory_pool_ops_t ops = { + .version = UMF_VERSION_CURRENT, .initialize = nullInitialize, .finalize = nullFinalize, .malloc = nullMalloc, @@ -80,17 +80,17 @@ uma_memory_pool_handle_t nullPoolCreate(void) { .free = nullFree, .get_last_allocation_error = nullGetLastStatus}; - uma_memory_provider_handle_t providerDesc = nullProviderCreate(); - uma_memory_pool_handle_t hPool; - enum uma_result_t ret = umaPoolCreate(&ops, &providerDesc, 1, NULL, &hPool); + umf_memory_provider_handle_t providerDesc = nullProviderCreate(); + umf_memory_pool_handle_t hPool; + enum umf_result_t ret = umfPoolCreate(&ops, &providerDesc, 1, NULL, &hPool); (void)ret; /* silence unused variable warning */ - assert(ret == UMA_RESULT_SUCCESS); + assert(ret == UMF_RESULT_SUCCESS); return hPool; } struct traceParams { - uma_memory_pool_handle_t hUpstreamPool; + umf_memory_pool_handle_t hUpstreamPool; void (*trace)(const char *); }; @@ -98,8 +98,8 @@ struct tracePool { struct traceParams params; }; -static enum uma_result_t -traceInitialize(uma_memory_provider_handle_t *providers, size_t numProviders, +static enum umf_result_t +traceInitialize(umf_memory_provider_handle_t *providers, size_t numProviders, void *params, void **pool) { struct tracePool *tracePool = (struct tracePool *)malloc(sizeof(struct tracePool)); @@ -110,7 +110,7 @@ traceInitialize(uma_memory_provider_handle_t *providers, size_t numProviders, assert(providers && numProviders); *pool = tracePool; - return UMA_RESULT_SUCCESS; + return UMF_RESULT_SUCCESS; } static void traceFinalize(void *pool) { free(pool); } @@ -119,28 +119,28 @@ static void *traceMalloc(void *pool, size_t size) { struct tracePool *tracePool = (struct tracePool *)pool; tracePool->params.trace("malloc"); - return umaPoolMalloc(tracePool->params.hUpstreamPool, size); + return umfPoolMalloc(tracePool->params.hUpstreamPool, size); } static void *traceCalloc(void *pool, size_t num, size_t size) { struct tracePool *tracePool = (struct tracePool *)pool; tracePool->params.trace("calloc"); - return umaPoolCalloc(tracePool->params.hUpstreamPool, num, size); + return umfPoolCalloc(tracePool->params.hUpstreamPool, num, size); } static void *traceRealloc(void *pool, void *ptr, size_t size) { struct tracePool *tracePool = (struct tracePool *)pool; tracePool->params.trace("realloc"); - return umaPoolRealloc(tracePool->params.hUpstreamPool, ptr, size); + return umfPoolRealloc(tracePool->params.hUpstreamPool, ptr, size); } static void *traceAlignedMalloc(void *pool, size_t size, size_t alignment) { struct tracePool *tracePool = (struct tracePool *)pool; tracePool->params.trace("aligned_malloc"); - return umaPoolAlignedMalloc(tracePool->params.hUpstreamPool, size, + return umfPoolAlignedMalloc(tracePool->params.hUpstreamPool, size, alignment); } @@ -148,29 +148,29 @@ static size_t traceMallocUsableSize(void *pool, void *ptr) { struct tracePool *tracePool = (struct tracePool *)pool; tracePool->params.trace("malloc_usable_size"); - return umaPoolMallocUsableSize(tracePool->params.hUpstreamPool, ptr); + return umfPoolMallocUsableSize(tracePool->params.hUpstreamPool, ptr); } static void traceFree(void *pool, void *ptr) { struct tracePool *tracePool = (struct tracePool *)pool; tracePool->params.trace("free"); - umaPoolFree(tracePool->params.hUpstreamPool, ptr); + umfPoolFree(tracePool->params.hUpstreamPool, ptr); } -enum uma_result_t traceGetLastStatus(void *pool) { +enum umf_result_t traceGetLastStatus(void *pool) { struct tracePool *tracePool = (struct tracePool *)pool; tracePool->params.trace("get_last_native_error"); - return umaPoolGetLastAllocationError(tracePool->params.hUpstreamPool); + return umfPoolGetLastAllocationError(tracePool->params.hUpstreamPool); } -uma_memory_pool_handle_t -tracePoolCreate(uma_memory_pool_handle_t hUpstreamPool, - uma_memory_provider_handle_t providerDesc, +umf_memory_pool_handle_t +tracePoolCreate(umf_memory_pool_handle_t hUpstreamPool, + umf_memory_provider_handle_t providerDesc, void (*trace)(const char *)) { - struct uma_memory_pool_ops_t ops = { - .version = UMA_VERSION_CURRENT, + struct umf_memory_pool_ops_t ops = { + .version = UMF_VERSION_CURRENT, .initialize = traceInitialize, .finalize = traceFinalize, .malloc = traceMalloc, @@ -184,11 +184,11 @@ tracePoolCreate(uma_memory_pool_handle_t hUpstreamPool, struct traceParams params = {.hUpstreamPool = hUpstreamPool, .trace = trace}; - uma_memory_pool_handle_t hPool; - enum uma_result_t ret = - umaPoolCreate(&ops, &providerDesc, 1, ¶ms, &hPool); + umf_memory_pool_handle_t hPool; + enum umf_result_t ret = + umfPoolCreate(&ops, &providerDesc, 1, ¶ms, &hPool); (void)ret; /* silence unused variable warning */ - assert(ret == UMA_RESULT_SUCCESS); + assert(ret == UMF_RESULT_SUCCESS); return hPool; } diff --git a/test/unified_memory_allocation/common/pool.h b/test/unified_malloc_framework/common/pool.h similarity index 53% rename from test/unified_memory_allocation/common/pool.h rename to test/unified_malloc_framework/common/pool.h index d9719ad458..69d4711e3e 100644 --- a/test/unified_memory_allocation/common/pool.h +++ b/test/unified_malloc_framework/common/pool.h @@ -3,23 +3,23 @@ // See LICENSE.TXT // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#ifndef UR_UMA_TEST_POOL_H -#define UR_UMA_TEST_POOL_H +#ifndef UR_UMF_TEST_POOL_H +#define UR_UMF_TEST_POOL_H -#include +#include #if defined(__cplusplus) extern "C" { #endif -uma_memory_pool_handle_t nullPoolCreate(void); -uma_memory_pool_handle_t -tracePoolCreate(uma_memory_pool_handle_t hUpstreamPool, - uma_memory_provider_handle_t providerDesc, +umf_memory_pool_handle_t nullPoolCreate(void); +umf_memory_pool_handle_t +tracePoolCreate(umf_memory_pool_handle_t hUpstreamPool, + umf_memory_provider_handle_t providerDesc, void (*trace)(const char *)); #if defined(__cplusplus) } #endif -#endif // UR_UMA_TEST_POOL_H +#endif // UR_UMF_TEST_POOL_H diff --git a/test/unified_memory_allocation/common/pool.hpp b/test/unified_malloc_framework/common/pool.hpp similarity index 69% rename from test/unified_memory_allocation/common/pool.hpp rename to test/unified_malloc_framework/common/pool.hpp index 6e1552140d..7ec62b8741 100644 --- a/test/unified_memory_allocation/common/pool.hpp +++ b/test/unified_malloc_framework/common/pool.hpp @@ -8,28 +8,28 @@ * */ -#ifndef UMA_TEST_POOL_HPP -#define UMA_TEST_POOL_HPP 1 +#ifndef UMF_TEST_POOL_HPP +#define UMF_TEST_POOL_HPP 1 #include -#include -#include +#include +#include #include #include #include "base.hpp" -#include "uma_helpers.hpp" +#include "umf_helpers.hpp" -namespace uma_test { +namespace umf_test { -auto wrapPoolUnique(uma_memory_pool_handle_t hPool) { - return uma::pool_unique_handle_t(hPool, &umaPoolDestroy); +auto wrapPoolUnique(umf_memory_pool_handle_t hPool) { + return umf::pool_unique_handle_t(hPool, &umfPoolDestroy); } struct pool_base { - uma_result_t initialize(uma_memory_provider_handle_t *, size_t) noexcept { - return UMA_RESULT_SUCCESS; + umf_result_t initialize(umf_memory_provider_handle_t *, size_t) noexcept { + return UMF_RESULT_SUCCESS; }; void *malloc(size_t size) noexcept { return nullptr; } void *calloc(size_t, size_t) noexcept { return nullptr; } @@ -37,8 +37,8 @@ struct pool_base { void *aligned_malloc(size_t, size_t) noexcept { return nullptr; } size_t malloc_usable_size(void *) noexcept { return 0; } void free(void *) noexcept {} - enum uma_result_t get_last_allocation_error() noexcept { - return UMA_RESULT_SUCCESS; + enum umf_result_t get_last_allocation_error() noexcept { + return UMF_RESULT_SUCCESS; } }; @@ -69,20 +69,20 @@ struct malloc_pool : public pool_base { }; struct proxy_pool : public pool_base { - uma_result_t initialize(uma_memory_provider_handle_t *providers, + umf_result_t initialize(umf_memory_provider_handle_t *providers, size_t numProviders) noexcept { this->provider = providers[0]; - return UMA_RESULT_SUCCESS; + return UMF_RESULT_SUCCESS; } void *malloc(size_t size) noexcept { return aligned_malloc(size, 0); } void *calloc(size_t num, size_t size) noexcept { void *ptr; - auto ret = umaMemoryProviderAlloc(provider, num * size, 0, &ptr); + auto ret = umfMemoryProviderAlloc(provider, num * size, 0, &ptr); memset(ptr, 0, num * size); if (ptr) { - EXPECT_EQ_NOEXCEPT(ret, UMA_RESULT_SUCCESS); + EXPECT_EQ_NOEXCEPT(ret, UMF_RESULT_SUCCESS); } return ptr; } @@ -92,9 +92,9 @@ struct proxy_pool : public pool_base { } void *aligned_malloc(size_t size, size_t alignment) noexcept { void *ptr; - auto ret = umaMemoryProviderAlloc(provider, size, alignment, &ptr); + auto ret = umfMemoryProviderAlloc(provider, size, alignment, &ptr); if (ptr) { - EXPECT_EQ_NOEXCEPT(ret, UMA_RESULT_SUCCESS); + EXPECT_EQ_NOEXCEPT(ret, UMF_RESULT_SUCCESS); } return ptr; } @@ -103,12 +103,12 @@ struct proxy_pool : public pool_base { return 0; } void free(void *ptr) noexcept { - auto ret = umaMemoryProviderFree(provider, ptr, 0); - EXPECT_EQ_NOEXCEPT(ret, UMA_RESULT_SUCCESS); + auto ret = umfMemoryProviderFree(provider, ptr, 0); + EXPECT_EQ_NOEXCEPT(ret, UMF_RESULT_SUCCESS); } - uma_memory_provider_handle_t provider; + umf_memory_provider_handle_t provider; }; -} // namespace uma_test +} // namespace umf_test -#endif /* UMA_TEST_POOL_HPP */ +#endif /* UMF_TEST_POOL_HPP */ diff --git a/test/unified_memory_allocation/common/provider.c b/test/unified_malloc_framework/common/provider.c similarity index 66% rename from test/unified_memory_allocation/common/provider.c rename to test/unified_malloc_framework/common/provider.c index a180705a58..8f9e946bfc 100644 --- a/test/unified_memory_allocation/common/provider.c +++ b/test/unified_malloc_framework/common/provider.c @@ -5,33 +5,33 @@ #include "provider.h" -#include +#include #include #include -static enum uma_result_t nullInitialize(void *params, void **pool) { +static enum umf_result_t nullInitialize(void *params, void **pool) { (void)params; *pool = NULL; - return UMA_RESULT_SUCCESS; + return UMF_RESULT_SUCCESS; } static void nullFinalize(void *pool) { (void)pool; } -static enum uma_result_t nullAlloc(void *provider, size_t size, +static enum umf_result_t nullAlloc(void *provider, size_t size, size_t alignment, void **ptr) { (void)provider; (void)size; (void)alignment; (void)ptr; - return UMA_RESULT_SUCCESS; + return UMF_RESULT_SUCCESS; } -static enum uma_result_t nullFree(void *provider, void *ptr, size_t size) { +static enum umf_result_t nullFree(void *provider, void *ptr, size_t size) { (void)provider; (void)ptr; (void)size; - return UMA_RESULT_SUCCESS; + return UMF_RESULT_SUCCESS; } static void nullGetLastError(void *provider, const char **ppMsg, @@ -41,36 +41,36 @@ static void nullGetLastError(void *provider, const char **ppMsg, (void)pError; } -static enum uma_result_t nullGetRecommendedPageSize(void *provider, size_t size, +static enum umf_result_t nullGetRecommendedPageSize(void *provider, size_t size, size_t *pageSize) { (void)provider; (void)size; (void)pageSize; - return UMA_RESULT_SUCCESS; + return UMF_RESULT_SUCCESS; } -static enum uma_result_t nullGetPageSize(void *provider, void *ptr, +static enum umf_result_t nullGetPageSize(void *provider, void *ptr, size_t *pageSize) { (void)provider; (void)ptr; (void)pageSize; - return UMA_RESULT_SUCCESS; + return UMF_RESULT_SUCCESS; } -static enum uma_result_t nullPurgeLazy(void *provider, void *ptr, size_t size) { +static enum umf_result_t nullPurgeLazy(void *provider, void *ptr, size_t size) { (void)provider; (void)ptr; (void)size; - return UMA_RESULT_SUCCESS; + return UMF_RESULT_SUCCESS; } -static enum uma_result_t nullPurgeForce(void *provider, void *ptr, +static enum umf_result_t nullPurgeForce(void *provider, void *ptr, size_t size) { (void)provider; (void)ptr; (void)size; - return UMA_RESULT_SUCCESS; + return UMF_RESULT_SUCCESS; } static const char *nullName(void *provider) { @@ -78,9 +78,9 @@ static const char *nullName(void *provider) { return "null"; } -uma_memory_provider_handle_t nullProviderCreate(void) { - struct uma_memory_provider_ops_t ops = { - .version = UMA_VERSION_CURRENT, +umf_memory_provider_handle_t nullProviderCreate(void) { + struct umf_memory_provider_ops_t ops = { + .version = UMF_VERSION_CURRENT, .initialize = nullInitialize, .finalize = nullFinalize, .alloc = nullAlloc, @@ -92,44 +92,44 @@ uma_memory_provider_handle_t nullProviderCreate(void) { .purge_force = nullPurgeForce, .get_name = nullName}; - uma_memory_provider_handle_t hProvider; - enum uma_result_t ret = umaMemoryProviderCreate(&ops, NULL, &hProvider); + umf_memory_provider_handle_t hProvider; + enum umf_result_t ret = umfMemoryProviderCreate(&ops, NULL, &hProvider); (void)ret; /* silence unused variable warning */ - assert(ret == UMA_RESULT_SUCCESS); + assert(ret == UMF_RESULT_SUCCESS); return hProvider; } struct traceParams { - uma_memory_provider_handle_t hUpstreamProvider; + umf_memory_provider_handle_t hUpstreamProvider; void (*trace)(const char *); }; -static enum uma_result_t traceInitialize(void *params, void **pool) { +static enum umf_result_t traceInitialize(void *params, void **pool) { struct traceParams *tracePool = (struct traceParams *)malloc(sizeof(struct traceParams)); *tracePool = *((struct traceParams *)params); *pool = tracePool; - return UMA_RESULT_SUCCESS; + return UMF_RESULT_SUCCESS; } static void traceFinalize(void *pool) { free(pool); } -static enum uma_result_t traceAlloc(void *provider, size_t size, +static enum umf_result_t traceAlloc(void *provider, size_t size, size_t alignment, void **ptr) { struct traceParams *traceProvider = (struct traceParams *)provider; traceProvider->trace("alloc"); - return umaMemoryProviderAlloc(traceProvider->hUpstreamProvider, size, + return umfMemoryProviderAlloc(traceProvider->hUpstreamProvider, size, alignment, ptr); } -static enum uma_result_t traceFree(void *provider, void *ptr, size_t size) { +static enum umf_result_t traceFree(void *provider, void *ptr, size_t size) { struct traceParams *traceProvider = (struct traceParams *)provider; traceProvider->trace("free"); - return umaMemoryProviderFree(traceProvider->hUpstreamProvider, ptr, size); + return umfMemoryProviderFree(traceProvider->hUpstreamProvider, ptr, size); } static void traceGetLastError(void *provider, const char **ppMsg, @@ -137,44 +137,44 @@ static void traceGetLastError(void *provider, const char **ppMsg, struct traceParams *traceProvider = (struct traceParams *)provider; traceProvider->trace("get_last_native_error"); - umaMemoryProviderGetLastNativeError(traceProvider->hUpstreamProvider, ppMsg, + umfMemoryProviderGetLastNativeError(traceProvider->hUpstreamProvider, ppMsg, pError); } -static enum uma_result_t +static enum umf_result_t traceGetRecommendedPageSize(void *provider, size_t size, size_t *pageSize) { struct traceParams *traceProvider = (struct traceParams *)provider; traceProvider->trace("get_recommended_page_size"); - return umaMemoryProviderGetRecommendedPageSize( + return umfMemoryProviderGetRecommendedPageSize( traceProvider->hUpstreamProvider, size, pageSize); } -static enum uma_result_t traceGetPageSize(void *provider, void *ptr, +static enum umf_result_t traceGetPageSize(void *provider, void *ptr, size_t *pageSize) { struct traceParams *traceProvider = (struct traceParams *)provider; traceProvider->trace("get_min_page_size"); - return umaMemoryProviderGetMinPageSize(traceProvider->hUpstreamProvider, + return umfMemoryProviderGetMinPageSize(traceProvider->hUpstreamProvider, ptr, pageSize); } -static enum uma_result_t tracePurgeLazy(void *provider, void *ptr, +static enum umf_result_t tracePurgeLazy(void *provider, void *ptr, size_t size) { struct traceParams *traceProvider = (struct traceParams *)provider; traceProvider->trace("purge_lazy"); - return umaMemoryProviderPurgeLazy(traceProvider->hUpstreamProvider, ptr, + return umfMemoryProviderPurgeLazy(traceProvider->hUpstreamProvider, ptr, size); } -static enum uma_result_t tracePurgeForce(void *provider, void *ptr, +static enum umf_result_t tracePurgeForce(void *provider, void *ptr, size_t size) { struct traceParams *traceProvider = (struct traceParams *)provider; traceProvider->trace("purge_force"); - return umaMemoryProviderPurgeForce(traceProvider->hUpstreamProvider, ptr, + return umfMemoryProviderPurgeForce(traceProvider->hUpstreamProvider, ptr, size); } @@ -182,14 +182,14 @@ static const char *traceName(void *provider) { struct traceParams *traceProvider = (struct traceParams *)provider; traceProvider->trace("name"); - return umaMemoryProviderGetName(traceProvider->hUpstreamProvider); + return umfMemoryProviderGetName(traceProvider->hUpstreamProvider); } -uma_memory_provider_handle_t -traceProviderCreate(uma_memory_provider_handle_t hUpstreamProvider, +umf_memory_provider_handle_t +traceProviderCreate(umf_memory_provider_handle_t hUpstreamProvider, void (*trace)(const char *)) { - struct uma_memory_provider_ops_t ops = { - .version = UMA_VERSION_CURRENT, + struct umf_memory_provider_ops_t ops = { + .version = UMF_VERSION_CURRENT, .initialize = traceInitialize, .finalize = traceFinalize, .alloc = traceAlloc, @@ -204,10 +204,10 @@ traceProviderCreate(uma_memory_provider_handle_t hUpstreamProvider, struct traceParams params = {.hUpstreamProvider = hUpstreamProvider, .trace = trace}; - uma_memory_provider_handle_t hProvider; - enum uma_result_t ret = umaMemoryProviderCreate(&ops, ¶ms, &hProvider); + umf_memory_provider_handle_t hProvider; + enum umf_result_t ret = umfMemoryProviderCreate(&ops, ¶ms, &hProvider); (void)ret; /* silence unused variable warning */ - assert(ret == UMA_RESULT_SUCCESS); + assert(ret == UMF_RESULT_SUCCESS); return hProvider; } diff --git a/test/unified_memory_allocation/common/provider.h b/test/unified_malloc_framework/common/provider.h similarity index 55% rename from test/unified_memory_allocation/common/provider.h rename to test/unified_malloc_framework/common/provider.h index d0ffdf8069..e8451c5dde 100644 --- a/test/unified_memory_allocation/common/provider.h +++ b/test/unified_malloc_framework/common/provider.h @@ -3,22 +3,22 @@ // See LICENSE.TXT // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#ifndef UR_UMA_TEST_PROVIDER_H -#define UR_UMA_TEST_PROVIDER_H +#ifndef UR_UMF_TEST_PROVIDER_H +#define UR_UMF_TEST_PROVIDER_H -#include +#include #if defined(__cplusplus) extern "C" { #endif -uma_memory_provider_handle_t nullProviderCreate(void); -uma_memory_provider_handle_t -traceProviderCreate(uma_memory_provider_handle_t hUpstreamProvider, +umf_memory_provider_handle_t nullProviderCreate(void); +umf_memory_provider_handle_t +traceProviderCreate(umf_memory_provider_handle_t hUpstreamProvider, void (*trace)(const char *)); #if defined(__cplusplus) } #endif -#endif // UR_UMA_TEST_PROVIDER_H +#endif // UR_UMF_TEST_PROVIDER_H diff --git a/test/unified_malloc_framework/common/provider.hpp b/test/unified_malloc_framework/common/provider.hpp new file mode 100644 index 0000000000..518b2b0528 --- /dev/null +++ b/test/unified_malloc_framework/common/provider.hpp @@ -0,0 +1,81 @@ +/* + * + * Copyright (C) 2023 Intel Corporation + * + * Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. + * See LICENSE.TXT + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#ifndef UMF_TEST_PROVIDER_HPP +#define UMF_TEST_PROVIDER_HPP 1 + +#include +#include + +#include + +#include "base.hpp" +#include "umf_helpers.hpp" + +namespace umf_test { + +auto wrapProviderUnique(umf_memory_provider_handle_t hProvider) { + return umf::provider_unique_handle_t(hProvider, &umfMemoryProviderDestroy); +} + +struct provider_base { + umf_result_t initialize() noexcept { return UMF_RESULT_SUCCESS; }; + enum umf_result_t alloc(size_t, size_t, void **) noexcept { + return UMF_RESULT_ERROR_UNKNOWN; + } + enum umf_result_t free(void *ptr, size_t size) noexcept { + return UMF_RESULT_ERROR_UNKNOWN; + } + void get_last_native_error(const char **, int32_t *) noexcept {} + enum umf_result_t get_recommended_page_size(size_t size, + size_t *pageSize) noexcept { + return UMF_RESULT_ERROR_UNKNOWN; + } + enum umf_result_t get_min_page_size(void *ptr, size_t *pageSize) noexcept { + return UMF_RESULT_ERROR_UNKNOWN; + } + enum umf_result_t purge_lazy(void *ptr, size_t size) noexcept { + return UMF_RESULT_ERROR_UNKNOWN; + } + enum umf_result_t purge_force(void *ptr, size_t size) noexcept { + return UMF_RESULT_ERROR_UNKNOWN; + } + const char *get_name() noexcept { return "base"; } +}; + +struct provider_malloc : public provider_base { + enum umf_result_t alloc(size_t size, size_t align, void **ptr) noexcept { + if (!align) { + align = 8; + } + +#ifdef _WIN32 + *ptr = _aligned_malloc(size, align); +#else + *ptr = ::aligned_alloc(align, size); +#endif + + return (*ptr) ? UMF_RESULT_SUCCESS + : UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + enum umf_result_t free(void *ptr, size_t) noexcept { +#ifdef _WIN32 + _aligned_free(ptr); +#else + ::free(ptr); +#endif + return UMF_RESULT_SUCCESS; + } + const char *get_name() noexcept { return "malloc"; } +}; + +} // namespace umf_test + +#endif /* UMF_TEST_PROVIDER_HPP */ diff --git a/test/unified_memory_allocation/memoryPool.hpp b/test/unified_malloc_framework/memoryPool.hpp similarity index 70% rename from test/unified_memory_allocation/memoryPool.hpp rename to test/unified_malloc_framework/memoryPool.hpp index c242419ba1..ab9cc3661c 100644 --- a/test/unified_memory_allocation/memoryPool.hpp +++ b/test/unified_malloc_framework/memoryPool.hpp @@ -9,13 +9,13 @@ #include #include -#ifndef UMA_TEST_MEMORY_POOL_OPS_HPP -#define UMA_TEST_MEMORY_POOL_OPS_HPP +#ifndef UMF_TEST_MEMORY_POOL_OPS_HPP +#define UMF_TEST_MEMORY_POOL_OPS_HPP -struct umaPoolTest : uma_test::test, +struct umfPoolTest : umf_test::test, ::testing::WithParamInterface< - std::function> { - umaPoolTest() : pool(nullptr, nullptr) {} + std::function> { + umfPoolTest() : pool(nullptr, nullptr) {} void SetUp() override { test::SetUp(); this->pool = makePool(); @@ -23,20 +23,20 @@ struct umaPoolTest : uma_test::test, void TearDown() override { test::TearDown(); } - uma::pool_unique_handle_t makePool() { + umf::pool_unique_handle_t makePool() { auto pool = this->GetParam()(); EXPECT_NE(pool, nullptr); return pool; } - uma::pool_unique_handle_t pool; + umf::pool_unique_handle_t pool; }; -struct umaMultiPoolTest : umaPoolTest { +struct umfMultiPoolTest : umfPoolTest { static constexpr auto numPools = 16; void SetUp() override { - umaPoolTest::SetUp(); + umfPoolTest::SetUp(); pools.emplace_back(std::move(pool)); for (size_t i = 1; i < numPools; i++) { @@ -44,20 +44,20 @@ struct umaMultiPoolTest : umaPoolTest { } } - void TearDown() override { umaPoolTest::TearDown(); } + void TearDown() override { umfPoolTest::TearDown(); } - std::vector pools; + std::vector pools; }; -TEST_P(umaPoolTest, allocFree) { +TEST_P(umfPoolTest, allocFree) { static constexpr size_t allocSize = 64; - auto *ptr = umaPoolMalloc(pool.get(), allocSize); + auto *ptr = umfPoolMalloc(pool.get(), allocSize); ASSERT_NE(ptr, nullptr); std::memset(ptr, 0, allocSize); - umaPoolFree(pool.get(), ptr); + umfPoolFree(pool.get(), ptr); } -TEST_P(umaPoolTest, pow2AlignedAlloc) { +TEST_P(umfPoolTest, pow2AlignedAlloc) { #ifdef _WIN32 // TODO: implement support for windows GTEST_SKIP(); @@ -71,7 +71,7 @@ TEST_P(umaPoolTest, pow2AlignedAlloc) { std::vector allocs; for (size_t alloc = 0; alloc < numAllocs; alloc++) { - auto *ptr = umaPoolAlignedMalloc(pool.get(), alignment, alignment); + auto *ptr = umfPoolAlignedMalloc(pool.get(), alignment, alignment); ASSERT_NE(ptr, nullptr); ASSERT_TRUE(reinterpret_cast(ptr) % alignment == 0); std::memset(ptr, 0, alignment); @@ -79,14 +79,14 @@ TEST_P(umaPoolTest, pow2AlignedAlloc) { } for (auto &ptr : allocs) { - umaPoolFree(pool.get(), ptr); + umfPoolFree(pool.get(), ptr); } } } // TODO: add similar tests for realloc/aligned_alloc, etc. // TODO: add multithreaded tests -TEST_P(umaMultiPoolTest, memoryTracking) { +TEST_P(umfMultiPoolTest, memoryTracking) { static constexpr int allocSizes[] = {1, 2, 4, 8, 16, 20, 32, 40, 64, 128, 1024, 4096}; static constexpr auto nAllocs = 256; @@ -97,29 +97,29 @@ TEST_P(umaMultiPoolTest, memoryTracking) { std::uniform_int_distribution poolsDist(0, static_cast(pools.size() - 1)); - std::vector> ptrs; + std::vector> ptrs; for (size_t i = 0; i < nAllocs; i++) { auto &pool = pools[poolsDist(g)]; auto size = allocSizes[allocSizesDist(g)]; - auto *ptr = umaPoolMalloc(pool.get(), size); + auto *ptr = umfPoolMalloc(pool.get(), size); ASSERT_NE(ptr, nullptr); ptrs.emplace_back(ptr, size, pool.get()); } for (auto [ptr, size, expectedPool] : ptrs) { - auto pool = umaPoolByPtr(ptr); + auto pool = umfPoolByPtr(ptr); ASSERT_EQ(pool, expectedPool); - pool = umaPoolByPtr(reinterpret_cast( + pool = umfPoolByPtr(reinterpret_cast( reinterpret_cast(ptr) + size - 1)); ASSERT_EQ(pool, expectedPool); } for (auto p : ptrs) { - umaFree(std::get<0>(p)); + umfFree(std::get<0>(p)); } } -#endif /* UMA_TEST_MEMORY_POOL_OPS_HPP */ +#endif /* UMF_TEST_MEMORY_POOL_OPS_HPP */ diff --git a/test/unified_malloc_framework/memoryPoolAPI.cpp b/test/unified_malloc_framework/memoryPoolAPI.cpp new file mode 100644 index 0000000000..35c307866e --- /dev/null +++ b/test/unified_malloc_framework/memoryPoolAPI.cpp @@ -0,0 +1,311 @@ +// Copyright (C) 2023 Intel Corporation +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +// See LICENSE.TXT +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// This file contains tests for UMF pool API + +#include "pool.h" +#include "pool.hpp" +#include "provider.h" +#include "provider.hpp" + +#include "memoryPool.hpp" +#include "umf/memory_provider.h" + +#include +#include +#include +#include + +using umf_test::test; + +TEST_F(test, memoryPoolTrace) { + static std::unordered_map poolCalls; + static std::unordered_map providerCalls; + auto tracePool = [](const char *name) { poolCalls[name]++; }; + auto traceProvider = [](const char *name) { providerCalls[name]++; }; + + auto nullProvider = umf_test::wrapProviderUnique(nullProviderCreate()); + auto tracingProvider = umf_test::wrapProviderUnique( + traceProviderCreate(nullProvider.get(), traceProvider)); + auto provider = tracingProvider.get(); + + auto [ret, proxyPool] = + umf::poolMakeUnique(&provider, 1); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + + umf_memory_provider_handle_t providerDesc = nullProviderCreate(); + auto tracingPool = umf_test::wrapPoolUnique( + tracePoolCreate(proxyPool.get(), providerDesc, tracePool)); + + size_t pool_call_count = 0; + size_t provider_call_count = 0; + + umfPoolMalloc(tracingPool.get(), 0); + ASSERT_EQ(poolCalls["malloc"], 1); + ASSERT_EQ(poolCalls.size(), ++pool_call_count); + + ASSERT_EQ(providerCalls["alloc"], 1); + ASSERT_EQ(providerCalls.size(), ++provider_call_count); + + umfPoolFree(tracingPool.get(), nullptr); + ASSERT_EQ(poolCalls["free"], 1); + ASSERT_EQ(poolCalls.size(), ++pool_call_count); + + ASSERT_EQ(providerCalls["free"], 1); + ASSERT_EQ(providerCalls.size(), ++provider_call_count); + + umfPoolCalloc(tracingPool.get(), 0, 0); + ASSERT_EQ(poolCalls["calloc"], 1); + ASSERT_EQ(poolCalls.size(), ++pool_call_count); + + ASSERT_EQ(providerCalls["alloc"], 2); + ASSERT_EQ(providerCalls.size(), provider_call_count); + + umfPoolRealloc(tracingPool.get(), nullptr, 0); + ASSERT_EQ(poolCalls["realloc"], 1); + ASSERT_EQ(poolCalls.size(), ++pool_call_count); + + ASSERT_EQ(providerCalls.size(), provider_call_count); + + umfPoolAlignedMalloc(tracingPool.get(), 0, 0); + ASSERT_EQ(poolCalls["aligned_malloc"], 1); + ASSERT_EQ(poolCalls.size(), ++pool_call_count); + + ASSERT_EQ(providerCalls["alloc"], 3); + ASSERT_EQ(providerCalls.size(), provider_call_count); + + umfPoolMallocUsableSize(tracingPool.get(), nullptr); + ASSERT_EQ(poolCalls["malloc_usable_size"], 1); + ASSERT_EQ(poolCalls.size(), ++pool_call_count); + + ASSERT_EQ(providerCalls.size(), provider_call_count); + + ret = umfPoolGetLastAllocationError(tracingPool.get()); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_EQ(poolCalls["get_last_native_error"], 1); + ASSERT_EQ(poolCalls.size(), ++pool_call_count); + + umfMemoryProviderDestroy(providerDesc); +} + +TEST_F(test, memoryPoolWithCustomProviders) { + umf_memory_provider_handle_t providers[] = {nullProviderCreate(), + nullProviderCreate()}; + + struct pool : public umf_test::pool_base { + umf_result_t initialize(umf_memory_provider_handle_t *providers, + size_t numProviders) noexcept { + EXPECT_NE_NOEXCEPT(providers, nullptr); + EXPECT_EQ_NOEXCEPT(numProviders, 2); + return UMF_RESULT_SUCCESS; + } + }; + + auto ret = umf::poolMakeUnique(providers, 2); + ASSERT_EQ(ret.first, UMF_RESULT_SUCCESS); + ASSERT_NE(ret.second, nullptr); + + for (auto &provider : providers) { + umfMemoryProviderDestroy(provider); + } +} + +TEST_F(test, retrieveMemoryProviders) { + static constexpr size_t numProviders = 4; + std::array providers = { + (umf_memory_provider_handle_t)0x1, (umf_memory_provider_handle_t)0x2, + (umf_memory_provider_handle_t)0x3, (umf_memory_provider_handle_t)0x4}; + + auto [ret, pool] = umf::poolMakeUnique( + providers.data(), numProviders); + + std::array retProviders; + size_t numProvidersRet = 0; + + ret = umfPoolGetMemoryProviders(pool.get(), 0, nullptr, &numProvidersRet); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_EQ(numProvidersRet, numProviders); + + ret = umfPoolGetMemoryProviders(pool.get(), numProviders, + retProviders.data(), nullptr); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_EQ(retProviders, providers); +} + +template +static auto +makePool(std::function makeProvider) { + auto providerUnique = makeProvider(); + umf_memory_provider_handle_t provider = providerUnique.get(); + auto pool = umf::poolMakeUnique(&provider, 1).second; + auto dtor = [provider = + providerUnique.release()](umf_memory_pool_handle_t hPool) { + umfPoolDestroy(hPool); + umfMemoryProviderDestroy(provider); + }; + return umf::pool_unique_handle_t(pool.release(), std::move(dtor)); +} + +INSTANTIATE_TEST_SUITE_P(mallocPoolTest, umfPoolTest, ::testing::Values([] { + return makePool([] { + return umf_test::wrapProviderUnique( + nullProviderCreate()); + }); + })); + +INSTANTIATE_TEST_SUITE_P( + mallocProviderPoolTest, umfPoolTest, ::testing::Values([] { + return makePool([] { + return umf::memoryProviderMakeUnique() + .second; + }); + })); + +INSTANTIATE_TEST_SUITE_P( + mallocMultiPoolTest, umfMultiPoolTest, ::testing::Values([] { + return makePool([] { + return umf::memoryProviderMakeUnique() + .second; + }); + })); + +////////////////// Negative test cases ///////////////// + +TEST_F(test, memoryPoolInvalidProvidersNullptr) { + auto ret = umf::poolMakeUnique(nullptr, 1); + ASSERT_EQ(ret.first, UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + +TEST_F(test, memoryPoolInvalidProvidersNum) { + auto nullProvider = umf_test::wrapProviderUnique(nullProviderCreate()); + umf_memory_provider_handle_t providers[] = {nullProvider.get()}; + + auto ret = umf::poolMakeUnique(providers, 0); + ASSERT_EQ(ret.first, UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + +struct poolInitializeTest : umf_test::test, + ::testing::WithParamInterface {}; + +INSTANTIATE_TEST_SUITE_P( + poolInitializeTest, poolInitializeTest, + ::testing::Values(UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY, + UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC, + UMF_RESULT_ERROR_INVALID_ARGUMENT, + UMF_RESULT_ERROR_UNKNOWN)); + +TEST_P(poolInitializeTest, errorPropagation) { + auto nullProvider = umf_test::wrapProviderUnique(nullProviderCreate()); + umf_memory_provider_handle_t providers[] = {nullProvider.get()}; + + struct pool : public umf_test::pool_base { + umf_result_t initialize(umf_memory_provider_handle_t *providers, + size_t numProviders, + umf_result_t errorToReturn) noexcept { + return errorToReturn; + } + }; + auto ret = umf::poolMakeUnique(providers, 1, this->GetParam()); + ASSERT_EQ(ret.first, this->GetParam()); + ASSERT_EQ(ret.second, nullptr); +} + +TEST_F(test, retrieveMemoryProvidersError) { + static constexpr size_t numProviders = 4; + std::array providers = { + (umf_memory_provider_handle_t)0x1, (umf_memory_provider_handle_t)0x2, + (umf_memory_provider_handle_t)0x3, (umf_memory_provider_handle_t)0x4}; + + auto [ret, pool] = umf::poolMakeUnique( + providers.data(), numProviders); + + ret = umfPoolGetMemoryProviders(pool.get(), 1, providers.data(), nullptr); + ASSERT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + +// TODO: extend test for different functions (not only alloc) +TEST_F(test, getLastFailedMemoryProvider) { + static constexpr size_t allocSize = 8; + static umf_result_t allocResult = UMF_RESULT_SUCCESS; + + struct memory_provider : public umf_test::provider_base { + umf_result_t initialize(const char *name) { + this->name = name; + return UMF_RESULT_SUCCESS; + } + + enum umf_result_t alloc(size_t size, size_t, void **ptr) noexcept { + if (allocResult == UMF_RESULT_SUCCESS) { + *ptr = malloc(size); + } else { + *ptr = nullptr; + } + + return allocResult; + } + + enum umf_result_t free(void *ptr, size_t size) noexcept { + ::free(ptr); + return UMF_RESULT_SUCCESS; + } + + const char *get_name() noexcept { return this->name; } + + const char *name; + }; + + auto [ret1, providerUnique1] = + umf::memoryProviderMakeUnique("provider1"); + ASSERT_EQ(ret1, UMF_RESULT_SUCCESS); + auto [ret2, providerUnique2] = + umf::memoryProviderMakeUnique("provider2"); + ASSERT_EQ(ret2, UMF_RESULT_SUCCESS); + + auto hProvider = providerUnique1.get(); + + auto [ret, pool] = umf::poolMakeUnique(&hProvider, 1); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + + ASSERT_EQ(umfGetLastFailedMemoryProvider(), nullptr); + auto ptr = umfPoolMalloc(pool.get(), allocSize); + ASSERT_NE(ptr, nullptr); + ASSERT_EQ(umfGetLastFailedMemoryProvider(), nullptr); + + // make provider return an error during allocation + allocResult = UMF_RESULT_ERROR_UNKNOWN; + ptr = umfPoolMalloc(pool.get(), allocSize); + ASSERT_EQ(ptr, nullptr); + ASSERT_EQ(std::string_view( + umfMemoryProviderGetName(umfGetLastFailedMemoryProvider())), + "provider1"); + + ret = umfMemoryProviderAlloc(providerUnique2.get(), allocSize, 0, &ptr); + ASSERT_EQ(ptr, nullptr); + ASSERT_EQ(std::string_view( + umfMemoryProviderGetName(umfGetLastFailedMemoryProvider())), + "provider2"); + + // succesfull provider should not be returned by umfGetLastFailedMemoryProvider + allocResult = UMF_RESULT_SUCCESS; + ptr = umfPoolMalloc(pool.get(), allocSize); + ASSERT_NE(ptr, nullptr); + ASSERT_EQ(std::string_view( + umfMemoryProviderGetName(umfGetLastFailedMemoryProvider())), + "provider2"); + + // erorr in another thread should not impact umfGetLastFailedMemoryProvider on this thread + allocResult = UMF_RESULT_ERROR_UNKNOWN; + std::thread t([&, hPool = pool.get()] { + ptr = umfPoolMalloc(hPool, allocSize); + ASSERT_EQ(ptr, nullptr); + ASSERT_EQ(std::string_view(umfMemoryProviderGetName( + umfGetLastFailedMemoryProvider())), + "provider1"); + }); + t.join(); + + ASSERT_EQ(std::string_view( + umfMemoryProviderGetName(umfGetLastFailedMemoryProvider())), + "provider2"); +} diff --git a/test/unified_memory_allocation/memoryProvider.hpp b/test/unified_malloc_framework/memoryProvider.hpp similarity index 60% rename from test/unified_memory_allocation/memoryProvider.hpp rename to test/unified_malloc_framework/memoryProvider.hpp index c7b5cc9f35..fe61ca39ae 100644 --- a/test/unified_memory_allocation/memoryProvider.hpp +++ b/test/unified_malloc_framework/memoryProvider.hpp @@ -8,23 +8,23 @@ #include #include -#ifndef UMA_TEST_MEMORY_PROVIDER_OPS_HPP -#define UMA_TEST_MEMORY_PROVIDER_OPS_HPP +#ifndef UMF_TEST_MEMORY_PROVIDER_OPS_HPP +#define UMF_TEST_MEMORY_PROVIDER_OPS_HPP -struct umaProviderTest - : uma_test::test, +struct umfProviderTest + : umf_test::test, ::testing::WithParamInterface()>> { - umaProviderTest() : provider(nullptr, nullptr) {} + std::pair()>> { + umfProviderTest() : provider(nullptr, nullptr) {} void SetUp() { test::SetUp(); auto [res, provider] = this->GetParam()(); - EXPECT_EQ(res, UMA_RESULT_SUCCESS); + EXPECT_EQ(res, UMF_RESULT_SUCCESS); EXPECT_NE(provider, nullptr); this->provider = std::move(provider); } void TearDown() override { test::TearDown(); } - uma::provider_unique_handle_t provider; + umf::provider_unique_handle_t provider; }; -#endif /* UMA_TEST_MEMORY_PROVIDER_OPS_HPP */ +#endif /* UMF_TEST_MEMORY_PROVIDER_OPS_HPP */ diff --git a/test/unified_memory_allocation/memoryProviderAPI.cpp b/test/unified_malloc_framework/memoryProviderAPI.cpp similarity index 58% rename from test/unified_memory_allocation/memoryProviderAPI.cpp rename to test/unified_malloc_framework/memoryProviderAPI.cpp index 754b86992e..fa02f9eb99 100644 --- a/test/unified_memory_allocation/memoryProviderAPI.cpp +++ b/test/unified_malloc_framework/memoryProviderAPI.cpp @@ -2,7 +2,7 @@ // Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. // See LICENSE.TXT // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// This file contains tests for UMA provider API +// This file contains tests for UMF provider API #include "pool.h" #include "provider.h" @@ -11,56 +11,56 @@ #include #include -using uma_test::test; +using umf_test::test; TEST_F(test, memoryProviderTrace) { static std::unordered_map calls; auto trace = [](const char *name) { calls[name]++; }; - auto nullProvider = uma_test::wrapProviderUnique(nullProviderCreate()); - auto tracingProvider = uma_test::wrapProviderUnique( + auto nullProvider = umf_test::wrapProviderUnique(nullProviderCreate()); + auto tracingProvider = umf_test::wrapProviderUnique( traceProviderCreate(nullProvider.get(), trace)); size_t call_count = 0; - auto ret = umaMemoryProviderAlloc(tracingProvider.get(), 0, 0, nullptr); - ASSERT_EQ(ret, UMA_RESULT_SUCCESS); + auto ret = umfMemoryProviderAlloc(tracingProvider.get(), 0, 0, nullptr); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); ASSERT_EQ(calls["alloc"], 1); ASSERT_EQ(calls.size(), ++call_count); - ret = umaMemoryProviderFree(tracingProvider.get(), nullptr, 0); - ASSERT_EQ(ret, UMA_RESULT_SUCCESS); + ret = umfMemoryProviderFree(tracingProvider.get(), nullptr, 0); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); ASSERT_EQ(calls["free"], 1); ASSERT_EQ(calls.size(), ++call_count); - umaMemoryProviderGetLastNativeError(tracingProvider.get(), nullptr, + umfMemoryProviderGetLastNativeError(tracingProvider.get(), nullptr, nullptr); ASSERT_EQ(calls["get_last_native_error"], 1); ASSERT_EQ(calls.size(), ++call_count); - ret = umaMemoryProviderGetRecommendedPageSize(tracingProvider.get(), 0, + ret = umfMemoryProviderGetRecommendedPageSize(tracingProvider.get(), 0, nullptr); - ASSERT_EQ(ret, UMA_RESULT_SUCCESS); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); ASSERT_EQ(calls["get_recommended_page_size"], 1); ASSERT_EQ(calls.size(), ++call_count); - ret = umaMemoryProviderGetMinPageSize(tracingProvider.get(), nullptr, + ret = umfMemoryProviderGetMinPageSize(tracingProvider.get(), nullptr, nullptr); - ASSERT_EQ(ret, UMA_RESULT_SUCCESS); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); ASSERT_EQ(calls["get_min_page_size"], 1); ASSERT_EQ(calls.size(), ++call_count); - ret = umaMemoryProviderPurgeLazy(tracingProvider.get(), nullptr, 0); - ASSERT_EQ(ret, UMA_RESULT_SUCCESS); + ret = umfMemoryProviderPurgeLazy(tracingProvider.get(), nullptr, 0); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); ASSERT_EQ(calls["purge_lazy"], 1); ASSERT_EQ(calls.size(), ++call_count); - ret = umaMemoryProviderPurgeForce(tracingProvider.get(), nullptr, 0); - ASSERT_EQ(ret, UMA_RESULT_SUCCESS); + ret = umfMemoryProviderPurgeForce(tracingProvider.get(), nullptr, 0); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); ASSERT_EQ(calls["purge_force"], 1); ASSERT_EQ(calls.size(), ++call_count); - const char *pName = umaMemoryProviderGetName(tracingProvider.get()); + const char *pName = umfMemoryProviderGetName(tracingProvider.get()); ASSERT_EQ(calls["name"], 1); ASSERT_EQ(calls.size(), ++call_count); ASSERT_EQ(std::string(pName), std::string("null")); @@ -69,23 +69,23 @@ TEST_F(test, memoryProviderTrace) { //////////////////////////// Negative test cases /////////////////////////////////// -struct providerInitializeTest : uma_test::test, - ::testing::WithParamInterface {}; +struct providerInitializeTest : umf_test::test, + ::testing::WithParamInterface {}; INSTANTIATE_TEST_SUITE_P( providerInitializeTest, providerInitializeTest, - ::testing::Values(UMA_RESULT_ERROR_OUT_OF_HOST_MEMORY, - UMA_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC, - UMA_RESULT_ERROR_INVALID_ARGUMENT, - UMA_RESULT_ERROR_UNKNOWN)); + ::testing::Values(UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY, + UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC, + UMF_RESULT_ERROR_INVALID_ARGUMENT, + UMF_RESULT_ERROR_UNKNOWN)); TEST_P(providerInitializeTest, errorPropagation) { - struct provider : public uma_test::provider_base { - uma_result_t initialize(uma_result_t errorToReturn) noexcept { + struct provider : public umf_test::provider_base { + umf_result_t initialize(umf_result_t errorToReturn) noexcept { return errorToReturn; } }; - auto ret = uma::memoryProviderMakeUnique(this->GetParam()); + auto ret = umf::memoryProviderMakeUnique(this->GetParam()); ASSERT_EQ(ret.first, this->GetParam()); ASSERT_EQ(ret.second, nullptr); } diff --git a/test/unified_memory_allocation/uma_pools/CMakeLists.txt b/test/unified_malloc_framework/umf_pools/CMakeLists.txt similarity index 52% rename from test/unified_memory_allocation/uma_pools/CMakeLists.txt rename to test/unified_malloc_framework/umf_pools/CMakeLists.txt index c38921d146..578059ed16 100644 --- a/test/unified_memory_allocation/uma_pools/CMakeLists.txt +++ b/test/unified_malloc_framework/umf_pools/CMakeLists.txt @@ -3,8 +3,8 @@ # See LICENSE.TXT # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -add_uma_test(disjointPool disjoint_pool.cpp) -add_uma_test(disjointPoolConfigParser disjoint_pool_config_parser.cpp) +add_umf_test(disjointPool disjoint_pool.cpp) +add_umf_test(disjointPoolConfigParser disjoint_pool_config_parser.cpp) -target_include_directories(uma_test-disjointPool PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/..) -target_link_libraries(uma_test-disjointPool PRIVATE ${PROJECT_NAME}::common) +target_include_directories(umf_test-disjointPool PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/..) +target_link_libraries(umf_test-disjointPool PRIVATE ${PROJECT_NAME}::common) diff --git a/test/unified_memory_allocation/uma_pools/disjoint_pool.cpp b/test/unified_malloc_framework/umf_pools/disjoint_pool.cpp similarity index 61% rename from test/unified_memory_allocation/uma_pools/disjoint_pool.cpp rename to test/unified_malloc_framework/umf_pools/disjoint_pool.cpp index 88b6d769d8..937c853119 100644 --- a/test/unified_memory_allocation/uma_pools/disjoint_pool.cpp +++ b/test/unified_malloc_framework/umf_pools/disjoint_pool.cpp @@ -25,21 +25,21 @@ static usm::DisjointPool::Config poolConfig() { static auto makePool() { auto [ret, providerUnique] = - uma::memoryProviderMakeUnique(); - EXPECT_EQ(ret, UMA_RESULT_SUCCESS); + umf::memoryProviderMakeUnique(); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); auto provider = providerUnique.release(); auto [retp, pool] = - uma::poolMakeUnique(&provider, 1, poolConfig()); - EXPECT_EQ(retp, UMA_RESULT_SUCCESS); - auto dtor = [provider = provider](uma_memory_pool_handle_t hPool) { - umaPoolDestroy(hPool); - umaMemoryProviderDestroy(provider); + umf::poolMakeUnique(&provider, 1, poolConfig()); + EXPECT_EQ(retp, UMF_RESULT_SUCCESS); + auto dtor = [provider = provider](umf_memory_pool_handle_t hPool) { + umfPoolDestroy(hPool); + umfMemoryProviderDestroy(provider); }; - return uma::pool_unique_handle_t(pool.release(), std::move(dtor)); + return umf::pool_unique_handle_t(pool.release(), std::move(dtor)); } -INSTANTIATE_TEST_SUITE_P(disjointPoolTests, umaPoolTest, +INSTANTIATE_TEST_SUITE_P(disjointPoolTests, umfPoolTest, ::testing::Values(makePool)); -INSTANTIATE_TEST_SUITE_P(disjointMultiPoolTests, umaMultiPoolTest, +INSTANTIATE_TEST_SUITE_P(disjointMultiPoolTests, umfMultiPoolTest, ::testing::Values(makePool)); diff --git a/test/unified_memory_allocation/uma_pools/disjoint_pool_config_parser.cpp b/test/unified_malloc_framework/umf_pools/disjoint_pool_config_parser.cpp similarity index 100% rename from test/unified_memory_allocation/uma_pools/disjoint_pool_config_parser.cpp rename to test/unified_malloc_framework/umf_pools/disjoint_pool_config_parser.cpp diff --git a/test/unified_memory_allocation/CMakeLists.txt b/test/unified_memory_allocation/CMakeLists.txt deleted file mode 100644 index 7cb5c1c8fc..0000000000 --- a/test/unified_memory_allocation/CMakeLists.txt +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (C) 2022-2023 Intel Corporation -# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. -# See LICENSE.TXT -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -set(UR_UMA_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}) - -add_library(uma_test_helpers STATIC - ${UR_UMA_TEST_DIR}/common/pool.c - ${UR_UMA_TEST_DIR}/common/provider.c -) - -target_link_libraries(uma_test_helpers PRIVATE ${PROJECT_NAME}::common) - -function(add_uma_test name) - set(TEST_TARGET_NAME uma_test-${name}) - add_executable(${TEST_TARGET_NAME} - ${ARGN}) - target_link_libraries(${TEST_TARGET_NAME} - PRIVATE - ${PROJECT_NAME}::unified_memory_allocation - ${PROJECT_NAME}::common - uma_test_helpers - GTest::gtest_main) - target_include_directories(${TEST_TARGET_NAME} PRIVATE - ${UR_UMA_TEST_DIR}/common) - add_test(NAME uma-${name} - COMMAND ${TEST_TARGET_NAME} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) - set_tests_properties(uma-${name} PROPERTIES LABELS "uma") -endfunction() - -add_subdirectory(uma_pools) - -add_uma_test(memoryProvider memoryProviderAPI.cpp) -add_uma_test(memoryPool memoryPoolAPI.cpp) -add_uma_test(base base.cpp) diff --git a/test/unified_memory_allocation/common/provider.hpp b/test/unified_memory_allocation/common/provider.hpp deleted file mode 100644 index bca4f66872..0000000000 --- a/test/unified_memory_allocation/common/provider.hpp +++ /dev/null @@ -1,81 +0,0 @@ -/* - * - * Copyright (C) 2023 Intel Corporation - * - * Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. - * See LICENSE.TXT - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - * - */ - -#ifndef UMA_TEST_PROVIDER_HPP -#define UMA_TEST_PROVIDER_HPP 1 - -#include -#include - -#include - -#include "base.hpp" -#include "uma_helpers.hpp" - -namespace uma_test { - -auto wrapProviderUnique(uma_memory_provider_handle_t hProvider) { - return uma::provider_unique_handle_t(hProvider, &umaMemoryProviderDestroy); -} - -struct provider_base { - uma_result_t initialize() noexcept { return UMA_RESULT_SUCCESS; }; - enum uma_result_t alloc(size_t, size_t, void **) noexcept { - return UMA_RESULT_ERROR_UNKNOWN; - } - enum uma_result_t free(void *ptr, size_t size) noexcept { - return UMA_RESULT_ERROR_UNKNOWN; - } - void get_last_native_error(const char **, int32_t *) noexcept {} - enum uma_result_t get_recommended_page_size(size_t size, - size_t *pageSize) noexcept { - return UMA_RESULT_ERROR_UNKNOWN; - } - enum uma_result_t get_min_page_size(void *ptr, size_t *pageSize) noexcept { - return UMA_RESULT_ERROR_UNKNOWN; - } - enum uma_result_t purge_lazy(void *ptr, size_t size) noexcept { - return UMA_RESULT_ERROR_UNKNOWN; - } - enum uma_result_t purge_force(void *ptr, size_t size) noexcept { - return UMA_RESULT_ERROR_UNKNOWN; - } - const char *get_name() noexcept { return "base"; } -}; - -struct provider_malloc : public provider_base { - enum uma_result_t alloc(size_t size, size_t align, void **ptr) noexcept { - if (!align) { - align = 8; - } - -#ifdef _WIN32 - *ptr = _aligned_malloc(size, align); -#else - *ptr = ::aligned_alloc(align, size); -#endif - - return (*ptr) ? UMA_RESULT_SUCCESS - : UMA_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - enum uma_result_t free(void *ptr, size_t) noexcept { -#ifdef _WIN32 - _aligned_free(ptr); -#else - ::free(ptr); -#endif - return UMA_RESULT_SUCCESS; - } - const char *get_name() noexcept { return "malloc"; } -}; - -} // namespace uma_test - -#endif /* UMA_TEST_PROVIDER_HPP */ diff --git a/test/unified_memory_allocation/memoryPoolAPI.cpp b/test/unified_memory_allocation/memoryPoolAPI.cpp deleted file mode 100644 index 015d3e1dd5..0000000000 --- a/test/unified_memory_allocation/memoryPoolAPI.cpp +++ /dev/null @@ -1,311 +0,0 @@ -// Copyright (C) 2023 Intel Corporation -// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. -// See LICENSE.TXT -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// This file contains tests for UMA pool API - -#include "pool.h" -#include "pool.hpp" -#include "provider.h" -#include "provider.hpp" - -#include "memoryPool.hpp" -#include "uma/memory_provider.h" - -#include -#include -#include -#include - -using uma_test::test; - -TEST_F(test, memoryPoolTrace) { - static std::unordered_map poolCalls; - static std::unordered_map providerCalls; - auto tracePool = [](const char *name) { poolCalls[name]++; }; - auto traceProvider = [](const char *name) { providerCalls[name]++; }; - - auto nullProvider = uma_test::wrapProviderUnique(nullProviderCreate()); - auto tracingProvider = uma_test::wrapProviderUnique( - traceProviderCreate(nullProvider.get(), traceProvider)); - auto provider = tracingProvider.get(); - - auto [ret, proxyPool] = - uma::poolMakeUnique(&provider, 1); - ASSERT_EQ(ret, UMA_RESULT_SUCCESS); - - uma_memory_provider_handle_t providerDesc = nullProviderCreate(); - auto tracingPool = uma_test::wrapPoolUnique( - tracePoolCreate(proxyPool.get(), providerDesc, tracePool)); - - size_t pool_call_count = 0; - size_t provider_call_count = 0; - - umaPoolMalloc(tracingPool.get(), 0); - ASSERT_EQ(poolCalls["malloc"], 1); - ASSERT_EQ(poolCalls.size(), ++pool_call_count); - - ASSERT_EQ(providerCalls["alloc"], 1); - ASSERT_EQ(providerCalls.size(), ++provider_call_count); - - umaPoolFree(tracingPool.get(), nullptr); - ASSERT_EQ(poolCalls["free"], 1); - ASSERT_EQ(poolCalls.size(), ++pool_call_count); - - ASSERT_EQ(providerCalls["free"], 1); - ASSERT_EQ(providerCalls.size(), ++provider_call_count); - - umaPoolCalloc(tracingPool.get(), 0, 0); - ASSERT_EQ(poolCalls["calloc"], 1); - ASSERT_EQ(poolCalls.size(), ++pool_call_count); - - ASSERT_EQ(providerCalls["alloc"], 2); - ASSERT_EQ(providerCalls.size(), provider_call_count); - - umaPoolRealloc(tracingPool.get(), nullptr, 0); - ASSERT_EQ(poolCalls["realloc"], 1); - ASSERT_EQ(poolCalls.size(), ++pool_call_count); - - ASSERT_EQ(providerCalls.size(), provider_call_count); - - umaPoolAlignedMalloc(tracingPool.get(), 0, 0); - ASSERT_EQ(poolCalls["aligned_malloc"], 1); - ASSERT_EQ(poolCalls.size(), ++pool_call_count); - - ASSERT_EQ(providerCalls["alloc"], 3); - ASSERT_EQ(providerCalls.size(), provider_call_count); - - umaPoolMallocUsableSize(tracingPool.get(), nullptr); - ASSERT_EQ(poolCalls["malloc_usable_size"], 1); - ASSERT_EQ(poolCalls.size(), ++pool_call_count); - - ASSERT_EQ(providerCalls.size(), provider_call_count); - - ret = umaPoolGetLastAllocationError(tracingPool.get()); - ASSERT_EQ(ret, UMA_RESULT_SUCCESS); - ASSERT_EQ(poolCalls["get_last_native_error"], 1); - ASSERT_EQ(poolCalls.size(), ++pool_call_count); - - umaMemoryProviderDestroy(providerDesc); -} - -TEST_F(test, memoryPoolWithCustomProviders) { - uma_memory_provider_handle_t providers[] = {nullProviderCreate(), - nullProviderCreate()}; - - struct pool : public uma_test::pool_base { - uma_result_t initialize(uma_memory_provider_handle_t *providers, - size_t numProviders) noexcept { - EXPECT_NE_NOEXCEPT(providers, nullptr); - EXPECT_EQ_NOEXCEPT(numProviders, 2); - return UMA_RESULT_SUCCESS; - } - }; - - auto ret = uma::poolMakeUnique(providers, 2); - ASSERT_EQ(ret.first, UMA_RESULT_SUCCESS); - ASSERT_NE(ret.second, nullptr); - - for (auto &provider : providers) { - umaMemoryProviderDestroy(provider); - } -} - -TEST_F(test, retrieveMemoryProviders) { - static constexpr size_t numProviders = 4; - std::array providers = { - (uma_memory_provider_handle_t)0x1, (uma_memory_provider_handle_t)0x2, - (uma_memory_provider_handle_t)0x3, (uma_memory_provider_handle_t)0x4}; - - auto [ret, pool] = uma::poolMakeUnique( - providers.data(), numProviders); - - std::array retProviders; - size_t numProvidersRet = 0; - - ret = umaPoolGetMemoryProviders(pool.get(), 0, nullptr, &numProvidersRet); - ASSERT_EQ(ret, UMA_RESULT_SUCCESS); - ASSERT_EQ(numProvidersRet, numProviders); - - ret = umaPoolGetMemoryProviders(pool.get(), numProviders, - retProviders.data(), nullptr); - ASSERT_EQ(ret, UMA_RESULT_SUCCESS); - ASSERT_EQ(retProviders, providers); -} - -template -static auto -makePool(std::function makeProvider) { - auto providerUnique = makeProvider(); - uma_memory_provider_handle_t provider = providerUnique.get(); - auto pool = uma::poolMakeUnique(&provider, 1).second; - auto dtor = [provider = - providerUnique.release()](uma_memory_pool_handle_t hPool) { - umaPoolDestroy(hPool); - umaMemoryProviderDestroy(provider); - }; - return uma::pool_unique_handle_t(pool.release(), std::move(dtor)); -} - -INSTANTIATE_TEST_SUITE_P(mallocPoolTest, umaPoolTest, ::testing::Values([] { - return makePool([] { - return uma_test::wrapProviderUnique( - nullProviderCreate()); - }); - })); - -INSTANTIATE_TEST_SUITE_P( - mallocProviderPoolTest, umaPoolTest, ::testing::Values([] { - return makePool([] { - return uma::memoryProviderMakeUnique() - .second; - }); - })); - -INSTANTIATE_TEST_SUITE_P( - mallocMultiPoolTest, umaMultiPoolTest, ::testing::Values([] { - return makePool([] { - return uma::memoryProviderMakeUnique() - .second; - }); - })); - -////////////////// Negative test cases ///////////////// - -TEST_F(test, memoryPoolInvalidProvidersNullptr) { - auto ret = uma::poolMakeUnique(nullptr, 1); - ASSERT_EQ(ret.first, UMA_RESULT_ERROR_INVALID_ARGUMENT); -} - -TEST_F(test, memoryPoolInvalidProvidersNum) { - auto nullProvider = uma_test::wrapProviderUnique(nullProviderCreate()); - uma_memory_provider_handle_t providers[] = {nullProvider.get()}; - - auto ret = uma::poolMakeUnique(providers, 0); - ASSERT_EQ(ret.first, UMA_RESULT_ERROR_INVALID_ARGUMENT); -} - -struct poolInitializeTest : uma_test::test, - ::testing::WithParamInterface {}; - -INSTANTIATE_TEST_SUITE_P( - poolInitializeTest, poolInitializeTest, - ::testing::Values(UMA_RESULT_ERROR_OUT_OF_HOST_MEMORY, - UMA_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC, - UMA_RESULT_ERROR_INVALID_ARGUMENT, - UMA_RESULT_ERROR_UNKNOWN)); - -TEST_P(poolInitializeTest, errorPropagation) { - auto nullProvider = uma_test::wrapProviderUnique(nullProviderCreate()); - uma_memory_provider_handle_t providers[] = {nullProvider.get()}; - - struct pool : public uma_test::pool_base { - uma_result_t initialize(uma_memory_provider_handle_t *providers, - size_t numProviders, - uma_result_t errorToReturn) noexcept { - return errorToReturn; - } - }; - auto ret = uma::poolMakeUnique(providers, 1, this->GetParam()); - ASSERT_EQ(ret.first, this->GetParam()); - ASSERT_EQ(ret.second, nullptr); -} - -TEST_F(test, retrieveMemoryProvidersError) { - static constexpr size_t numProviders = 4; - std::array providers = { - (uma_memory_provider_handle_t)0x1, (uma_memory_provider_handle_t)0x2, - (uma_memory_provider_handle_t)0x3, (uma_memory_provider_handle_t)0x4}; - - auto [ret, pool] = uma::poolMakeUnique( - providers.data(), numProviders); - - ret = umaPoolGetMemoryProviders(pool.get(), 1, providers.data(), nullptr); - ASSERT_EQ(ret, UMA_RESULT_ERROR_INVALID_ARGUMENT); -} - -// TODO: extend test for different functions (not only alloc) -TEST_F(test, getLastFailedMemoryProvider) { - static constexpr size_t allocSize = 8; - static uma_result_t allocResult = UMA_RESULT_SUCCESS; - - struct memory_provider : public uma_test::provider_base { - uma_result_t initialize(const char *name) { - this->name = name; - return UMA_RESULT_SUCCESS; - } - - enum uma_result_t alloc(size_t size, size_t, void **ptr) noexcept { - if (allocResult == UMA_RESULT_SUCCESS) { - *ptr = malloc(size); - } else { - *ptr = nullptr; - } - - return allocResult; - } - - enum uma_result_t free(void *ptr, size_t size) noexcept { - ::free(ptr); - return UMA_RESULT_SUCCESS; - } - - const char *get_name() noexcept { return this->name; } - - const char *name; - }; - - auto [ret1, providerUnique1] = - uma::memoryProviderMakeUnique("provider1"); - ASSERT_EQ(ret1, UMA_RESULT_SUCCESS); - auto [ret2, providerUnique2] = - uma::memoryProviderMakeUnique("provider2"); - ASSERT_EQ(ret2, UMA_RESULT_SUCCESS); - - auto hProvider = providerUnique1.get(); - - auto [ret, pool] = uma::poolMakeUnique(&hProvider, 1); - ASSERT_EQ(ret, UMA_RESULT_SUCCESS); - - ASSERT_EQ(umaGetLastFailedMemoryProvider(), nullptr); - auto ptr = umaPoolMalloc(pool.get(), allocSize); - ASSERT_NE(ptr, nullptr); - ASSERT_EQ(umaGetLastFailedMemoryProvider(), nullptr); - - // make provider return an error during allocation - allocResult = UMA_RESULT_ERROR_UNKNOWN; - ptr = umaPoolMalloc(pool.get(), allocSize); - ASSERT_EQ(ptr, nullptr); - ASSERT_EQ(std::string_view( - umaMemoryProviderGetName(umaGetLastFailedMemoryProvider())), - "provider1"); - - ret = umaMemoryProviderAlloc(providerUnique2.get(), allocSize, 0, &ptr); - ASSERT_EQ(ptr, nullptr); - ASSERT_EQ(std::string_view( - umaMemoryProviderGetName(umaGetLastFailedMemoryProvider())), - "provider2"); - - // succesfull provider should not be returned by umaGetLastFailedMemoryProvider - allocResult = UMA_RESULT_SUCCESS; - ptr = umaPoolMalloc(pool.get(), allocSize); - ASSERT_NE(ptr, nullptr); - ASSERT_EQ(std::string_view( - umaMemoryProviderGetName(umaGetLastFailedMemoryProvider())), - "provider2"); - - // erorr in another thread should not impact umaGetLastFailedMemoryProvider on this thread - allocResult = UMA_RESULT_ERROR_UNKNOWN; - std::thread t([&, hPool = pool.get()] { - ptr = umaPoolMalloc(hPool, allocSize); - ASSERT_EQ(ptr, nullptr); - ASSERT_EQ(std::string_view(umaMemoryProviderGetName( - umaGetLastFailedMemoryProvider())), - "provider1"); - }); - t.join(); - - ASSERT_EQ(std::string_view( - umaMemoryProviderGetName(umaGetLastFailedMemoryProvider())), - "provider2"); -} diff --git a/test/usm/usmPoolManager.cpp b/test/usm/usmPoolManager.cpp index b244a89c83..eaf44e119d 100644 --- a/test/usm/usmPoolManager.cpp +++ b/test/usm/usmPoolManager.cpp @@ -3,8 +3,8 @@ // See LICENSE.TXT // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "../unified_memory_allocation/common/pool.hpp" -#include "../unified_memory_allocation/common/provider.hpp" +#include "../unified_malloc_framework/common/pool.hpp" +#include "../unified_malloc_framework/common/provider.hpp" #include "ur_pool_manager.hpp" #include