From fa9e90f5d23312587b3a17920941334e0d1a58a1 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Thu, 9 May 2024 06:35:54 -0500 Subject: [PATCH] [Reland][Libomptarget] Statically link all plugin runtimes (#87009) This patch overhauls the `libomptarget` and plugin interface. Currently, we define a C API and compile each plugin as a separate shared library. Then, `libomptarget` loads these API functions and forwards its internal calls to them. This was originally designed to allow multiple implementations of a library to be live. However, since then no one has used this functionality and it prevents us from using much nicer interfaces. If the old behavior is desired it should instead be implemented as a separate plugin. This patch replaces the `PluginAdaptorTy` interface with the `GenericPluginTy` that is used by the plugins. Each plugin exports a `createPlugin_` function that is used to get the specific implementation. This code is now shared with `libomptarget`. There are some notable improvements to this. 1. Massively improved lifetimes of life runtime objects 2. The plugins can use a C++ interface 3. Global state does not need to be duplicated for each plugin + libomptarget 4. Easier to use and add features and improve error handling 5. Less function call overhead / Improved LTO performance. Additional changes in this plugin are related to contending with the fact that state is now shared. Initialization and deinitialization is now handled correctly and in phase with the underlying runtime, allowing us to actually know when something is getting deallocated. Depends on https://github.com/llvm/llvm-project/pull/86971 https://github.com/llvm/llvm-project/pull/86875 https://github.com/llvm/llvm-project/pull/86868 --- clang/test/Driver/linker-wrapper-image.c | 2 +- .../Frontend/Offloading/OffloadWrapper.cpp | 7 +- offload/include/PluginManager.h | 61 ++---- offload/include/device.h | 8 +- offload/plugins-nextgen/CMakeLists.txt | 19 +- offload/plugins-nextgen/amdgpu/CMakeLists.txt | 5 - offload/plugins-nextgen/amdgpu/src/rtl.cpp | 14 +- offload/plugins-nextgen/common/CMakeLists.txt | 4 +- .../common/include/PluginInterface.h | 94 +------- .../common/include/Utils/ELF.h | 2 - offload/plugins-nextgen/common/src/JIT.cpp | 40 ++-- .../common/src/PluginInterface.cpp | 205 ------------------ offload/plugins-nextgen/cuda/CMakeLists.txt | 5 - offload/plugins-nextgen/cuda/src/rtl.cpp | 14 +- offload/plugins-nextgen/host/CMakeLists.txt | 8 - offload/plugins-nextgen/host/src/rtl.cpp | 14 +- offload/src/CMakeLists.txt | 4 + offload/src/OffloadRTL.cpp | 1 + offload/src/OpenMP/InteropAPI.cpp | 4 +- offload/src/PluginManager.cpp | 129 ++++------- offload/src/device.cpp | 3 +- offload/src/interface.cpp | 2 - .../kernelreplay/llvm-omp-kernel-replay.cpp | 2 - .../unittests/Plugins/NextgenPluginsTest.cpp | 1 - 24 files changed, 125 insertions(+), 523 deletions(-) diff --git a/clang/test/Driver/linker-wrapper-image.c b/clang/test/Driver/linker-wrapper-image.c index d01445e3aed04e..5d5d62805e174d 100644 --- a/clang/test/Driver/linker-wrapper-image.c +++ b/clang/test/Driver/linker-wrapper-image.c @@ -30,8 +30,8 @@ // OPENMP: define internal void @.omp_offloading.descriptor_reg() section ".text.startup" { // OPENMP-NEXT: entry: -// OPENMP-NEXT: %0 = call i32 @atexit(ptr @.omp_offloading.descriptor_unreg) // OPENMP-NEXT: call void @__tgt_register_lib(ptr @.omp_offloading.descriptor) +// OPENMP-NEXT: %0 = call i32 @atexit(ptr @.omp_offloading.descriptor_unreg) // OPENMP-NEXT: ret void // OPENMP-NEXT: } diff --git a/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp b/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp index 7241d15ed1c670..8b6f9ea1f4cca3 100644 --- a/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp +++ b/llvm/lib/Frontend/Offloading/OffloadWrapper.cpp @@ -232,12 +232,13 @@ void createRegisterFunction(Module &M, GlobalVariable *BinDesc, // Construct function body IRBuilder<> Builder(BasicBlock::Create(C, "entry", Func)); + Builder.CreateCall(RegFuncC, BinDesc); + // Register the destructors with 'atexit'. This is expected by the CUDA // runtime and ensures that we clean up before dynamic objects are destroyed. - // This needs to be done before the runtime is called and registers its own. + // This needs to be done after plugin initialization to ensure that it is + // called before the plugin runtime is destroyed. Builder.CreateCall(AtExit, UnregFunc); - - Builder.CreateCall(RegFuncC, BinDesc); Builder.CreateRetVoid(); // Add this function to constructors. diff --git a/offload/include/PluginManager.h b/offload/include/PluginManager.h index eece7525e25e72..1d6804da75d928 100644 --- a/offload/include/PluginManager.h +++ b/offload/include/PluginManager.h @@ -13,10 +13,11 @@ #ifndef OMPTARGET_PLUGIN_MANAGER_H #define OMPTARGET_PLUGIN_MANAGER_H +#include "PluginInterface.h" + #include "DeviceImage.h" #include "ExclusiveAccess.h" #include "Shared/APITypes.h" -#include "Shared/PluginAPI.h" #include "Shared/Requirements.h" #include "device.h" @@ -34,38 +35,7 @@ #include #include -struct PluginManager; - -/// Plugin adaptors should be created via `PluginAdaptorTy::create` which will -/// invoke the constructor and call `PluginAdaptorTy::init`. Eventual errors are -/// reported back to the caller, otherwise a valid and initialized adaptor is -/// returned. -struct PluginAdaptorTy { - /// Try to create a plugin adaptor from a filename. - static llvm::Expected> - create(const std::string &Name); - - /// Name of the shared object file representing the plugin. - std::string Name; - - /// Access to the shared object file representing the plugin. - std::unique_ptr LibraryHandler; - -#define PLUGIN_API_HANDLE(NAME) \ - using NAME##_ty = decltype(__tgt_rtl_##NAME); \ - NAME##_ty *NAME = nullptr; - -#include "Shared/PluginAPI.inc" -#undef PLUGIN_API_HANDLE - - /// Create a plugin adaptor for filename \p Name with a dynamic library \p DL. - PluginAdaptorTy(const std::string &Name, - std::unique_ptr DL); - - /// Initialize the plugin adaptor, this can fail in which case the adaptor is - /// useless. - llvm::Error init(); -}; +using GenericPluginTy = llvm::omp::target::plugin::GenericPluginTy; /// Struct for the data required to handle plugins struct PluginManager { @@ -80,6 +50,8 @@ struct PluginManager { void init(); + void deinit(); + // Register a shared library with all (compatible) RTLs. void registerLib(__tgt_bin_desc *Desc); @@ -92,10 +64,9 @@ struct PluginManager { std::make_unique(TgtBinDesc, TgtDeviceImage)); } - /// Initialize as many devices as possible for this plugin adaptor. Devices - /// that fail to initialize are ignored. Returns the offset the devices were - /// registered at. - void initDevices(PluginAdaptorTy &RTL); + /// Initialize as many devices as possible for this plugin. Devices that fail + /// to initialize are ignored. + void initDevices(GenericPluginTy &RTL); /// Return the device presented to the user as device \p DeviceNo if it is /// initialized and ready. Otherwise return an error explaining the problem. @@ -151,8 +122,8 @@ struct PluginManager { // Initialize all plugins. void initAllPlugins(); - /// Iterator range for all plugin adaptors (in use or not, but always valid). - auto pluginAdaptors() { return llvm::make_pointee_range(PluginAdaptors); } + /// Iterator range for all plugins (in use or not, but always valid). + auto plugins() { return llvm::make_pointee_range(Plugins); } /// Return the user provided requirements. int64_t getRequirements() const { return Requirements.getRequirements(); } @@ -164,14 +135,14 @@ struct PluginManager { bool RTLsLoaded = false; llvm::SmallVector<__tgt_bin_desc *> DelayedBinDesc; - // List of all plugin adaptors, in use or not. - llvm::SmallVector> PluginAdaptors; + // List of all plugins, in use or not. + llvm::SmallVector> Plugins; - // Mapping of plugin adaptors to offsets in the device table. - llvm::DenseMap DeviceOffsets; + // Mapping of plugins to offsets in the device table. + llvm::DenseMap DeviceOffsets; - // Mapping of plugin adaptors to the number of used devices. - llvm::DenseMap DeviceUsed; + // Mapping of plugins to the number of used devices. + llvm::DenseMap DeviceUsed; // Set of all device images currently in use. llvm::DenseSet UsedImages; diff --git a/offload/include/device.h b/offload/include/device.h index bd2829722bb324..fd6e5fba5fc530 100644 --- a/offload/include/device.h +++ b/offload/include/device.h @@ -33,17 +33,19 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" +#include "PluginInterface.h" +using GenericPluginTy = llvm::omp::target::plugin::GenericPluginTy; + // Forward declarations. -struct PluginAdaptorTy; struct __tgt_bin_desc; struct __tgt_target_table; struct DeviceTy { int32_t DeviceID; - PluginAdaptorTy *RTL; + GenericPluginTy *RTL; int32_t RTLDeviceID; - DeviceTy(PluginAdaptorTy *RTL, int32_t DeviceID, int32_t RTLDeviceID); + DeviceTy(GenericPluginTy *RTL, int32_t DeviceID, int32_t RTLDeviceID); // DeviceTy is not copyable DeviceTy(const DeviceTy &D) = delete; DeviceTy &operator=(const DeviceTy &D) = delete; diff --git a/offload/plugins-nextgen/CMakeLists.txt b/offload/plugins-nextgen/CMakeLists.txt index df625e97c7ebf5..d1079f8a3e9cc2 100644 --- a/offload/plugins-nextgen/CMakeLists.txt +++ b/offload/plugins-nextgen/CMakeLists.txt @@ -14,7 +14,7 @@ set(common_dir ${CMAKE_CURRENT_SOURCE_DIR}/common) add_subdirectory(common) function(add_target_library target_name lib_name) - add_llvm_library(${target_name} SHARED + add_llvm_library(${target_name} STATIC LINK_COMPONENTS ${LLVM_TARGETS_TO_BUILD} AggressiveInstCombine @@ -46,27 +46,14 @@ function(add_target_library target_name lib_name) ) llvm_update_compile_flags(${target_name}) + target_include_directories(${target_name} PUBLIC ${common_dir}/include) target_link_libraries(${target_name} PRIVATE PluginCommon ${OPENMP_PTHREAD_LIB}) target_compile_definitions(${target_name} PRIVATE TARGET_NAME=${lib_name}) target_compile_definitions(${target_name} PRIVATE DEBUG_PREFIX="TARGET ${lib_name} RTL") - - if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") - # On FreeBSD, the 'environ' symbol is undefined at link time, but resolved by - # the dynamic linker at runtime. Therefore, allow the symbol to be undefined - # when creating a shared library. - target_link_libraries(${target_name} PRIVATE "-Wl,--allow-shlib-undefined") - else() - target_link_libraries(${target_name} PRIVATE "-Wl,-z,defs") - endif() - - if(LIBOMP_HAVE_VERSION_SCRIPT_FLAG) - target_link_libraries(${target_name} PRIVATE - "-Wl,--version-script=${common_dir}/../exports") - endif() - set_target_properties(${target_name} PROPERTIES CXX_VISIBILITY_PRESET protected) + set_target_properties(${target_name} PROPERTIES POSITION_INDEPENDENT_CODE ON) endfunction() foreach(plugin IN LISTS LIBOMPTARGET_PLUGINS_TO_BUILD) diff --git a/offload/plugins-nextgen/amdgpu/CMakeLists.txt b/offload/plugins-nextgen/amdgpu/CMakeLists.txt index f5f7096137c20f..738183f8945ed7 100644 --- a/offload/plugins-nextgen/amdgpu/CMakeLists.txt +++ b/offload/plugins-nextgen/amdgpu/CMakeLists.txt @@ -57,8 +57,3 @@ else() libomptarget_say("Not generating AMDGPU tests, no supported devices detected." " Use 'LIBOMPTARGET_FORCE_AMDGPU_TESTS' to override.") endif() - -# Install plugin under the lib destination folder. -install(TARGETS omptarget.rtl.amdgpu LIBRARY DESTINATION "${OFFLOAD_INSTALL_LIBDIR}") -set_target_properties(omptarget.rtl.amdgpu PROPERTIES - INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..") diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp index 00650b801b4202..295685fceaa438 100644 --- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp +++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp @@ -3064,10 +3064,6 @@ struct AMDGPUPluginTy final : public GenericPluginTy { // HSA functions from now on, e.g., hsa_shut_down. Initialized = true; -#ifdef OMPT_SUPPORT - ompt::connectLibrary(); -#endif - // Register event handler to detect memory errors on the devices. Status = hsa_amd_register_system_event_handler(eventHandler, nullptr); if (auto Err = Plugin::check( @@ -3155,6 +3151,8 @@ struct AMDGPUPluginTy final : public GenericPluginTy { Triple::ArchType getTripleArch() const override { return Triple::amdgcn; } + const char *getName() const override { return GETNAME(TARGET_NAME); } + /// Get the ELF code for recognizing the compatible image binary. uint16_t getMagicElfBits() const override { return ELF::EM_AMDGPU; } @@ -3387,8 +3385,6 @@ Error AMDGPUKernelTy::printLaunchInfoDetails(GenericDeviceTy &GenericDevice, return Plugin::success(); } -GenericPluginTy *PluginTy::createPlugin() { return new AMDGPUPluginTy(); } - template static Error Plugin::check(int32_t Code, const char *ErrFmt, ArgsTy... Args) { hsa_status_t ResultCode = static_cast(Code); @@ -3476,3 +3472,9 @@ void *AMDGPUDeviceTy::allocate(size_t Size, void *, TargetAllocTy Kind) { } // namespace target } // namespace omp } // namespace llvm + +extern "C" { +llvm::omp::target::plugin::GenericPluginTy *createPlugin_amdgpu() { + return new llvm::omp::target::plugin::AMDGPUPluginTy(); +} +} diff --git a/offload/plugins-nextgen/common/CMakeLists.txt b/offload/plugins-nextgen/common/CMakeLists.txt index acf0af63f0508c..a470dcee6d8598 100644 --- a/offload/plugins-nextgen/common/CMakeLists.txt +++ b/offload/plugins-nextgen/common/CMakeLists.txt @@ -66,6 +66,4 @@ target_include_directories(PluginCommon PUBLIC ${LIBOMPTARGET_INCLUDE_DIR} ) -set_target_properties(PluginCommon PROPERTIES - POSITION_INDEPENDENT_CODE ON - CXX_VISIBILITY_PRESET protected) +set_target_properties(PluginCommon PROPERTIES POSITION_INDEPENDENT_CODE ON) diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h index 79e8464bfda5c1..e7a008f3a85730 100644 --- a/offload/plugins-nextgen/common/include/PluginInterface.h +++ b/offload/plugins-nextgen/common/include/PluginInterface.h @@ -1010,6 +1010,9 @@ struct GenericPluginTy { /// Get the target triple of this plugin. virtual Triple::ArchType getTripleArch() const = 0; + /// Get the constant name identifier for this plugin. + virtual const char *getName() const = 0; + /// Allocate a structure using the internal allocator. template Ty *allocate() { return reinterpret_cast(Allocator.Allocate(sizeof(Ty), alignof(Ty))); @@ -1226,7 +1229,7 @@ namespace Plugin { /// Create a success error. This is the same as calling Error::success(), but /// it is recommended to use this one for consistency with Plugin::error() and /// Plugin::check(). -static Error success() { return Error::success(); } +static inline Error success() { return Error::success(); } /// Create a string error. template @@ -1246,95 +1249,6 @@ template static Error check(int32_t ErrorCode, const char *ErrFmt, ArgsTy... Args); } // namespace Plugin -/// Class for simplifying the getter operation of the plugin. Anywhere on the -/// code, the current plugin can be retrieved by Plugin::get(). The class also -/// declares functions to create plugin-specific object instances. The check(), -/// createPlugin(), createDevice() and createGlobalHandler() functions should be -/// defined by each plugin implementation. -class PluginTy { - // Reference to the plugin instance. - static GenericPluginTy *SpecificPlugin; - - PluginTy() { - if (auto Err = init()) - REPORT("Failed to initialize plugin: %s\n", - toString(std::move(Err)).data()); - } - - ~PluginTy() { - if (auto Err = deinit()) - REPORT("Failed to deinitialize plugin: %s\n", - toString(std::move(Err)).data()); - } - - PluginTy(const PluginTy &) = delete; - void operator=(const PluginTy &) = delete; - - /// Create and intialize the plugin instance. - static Error init() { - assert(!SpecificPlugin && "Plugin already created"); - - // Create the specific plugin. - SpecificPlugin = createPlugin(); - assert(SpecificPlugin && "Plugin was not created"); - - // Initialize the plugin. - return SpecificPlugin->init(); - } - - // Deinitialize and destroy the plugin instance. - static Error deinit() { - assert(SpecificPlugin && "Plugin no longer valid"); - - for (int32_t DevNo = 0, NumDev = SpecificPlugin->getNumDevices(); - DevNo < NumDev; ++DevNo) - if (auto Err = SpecificPlugin->deinitDevice(DevNo)) - return Err; - - // Deinitialize the plugin. - if (auto Err = SpecificPlugin->deinit()) - return Err; - - // Delete the plugin instance. - delete SpecificPlugin; - - // Invalidate the plugin reference. - SpecificPlugin = nullptr; - - return Plugin::success(); - } - -public: - /// Initialize the plugin if needed. The plugin could have been initialized by - /// a previous call to Plugin::get(). - static Error initIfNeeded() { - // Trigger the initialization if needed. - get(); - - return Error::success(); - } - - /// Get a reference (or create if it was not created) to the plugin instance. - static GenericPluginTy &get() { - // This static variable will initialize the underlying plugin instance in - // case there was no previous explicit initialization. The initialization is - // thread safe. - static PluginTy Plugin; - - assert(SpecificPlugin && "Plugin is not active"); - return *SpecificPlugin; - } - - /// Get a reference to the plugin with a specific plugin-specific type. - template static Ty &get() { return static_cast(get()); } - - /// Indicate whether the plugin is active. - static bool isActive() { return SpecificPlugin != nullptr; } - - /// Create a plugin instance. - static GenericPluginTy *createPlugin(); -}; - /// Auxiliary interface class for GenericDeviceResourceManagerTy. This class /// acts as a reference to a device resource, such as a stream, and requires /// some basic functions to be implemented. The derived class should define an diff --git a/offload/plugins-nextgen/common/include/Utils/ELF.h b/offload/plugins-nextgen/common/include/Utils/ELF.h index f87e0a5ed02b4c..dcfdb5bd7b0352 100644 --- a/offload/plugins-nextgen/common/include/Utils/ELF.h +++ b/offload/plugins-nextgen/common/include/Utils/ELF.h @@ -13,8 +13,6 @@ #ifndef LLVM_OPENMP_LIBOMPTARGET_PLUGINS_ELF_UTILS_H #define LLVM_OPENMP_LIBOMPTARGET_PLUGINS_ELF_UTILS_H -#include "Shared/PluginAPI.h" - #include "llvm/Object/ELF.h" #include "llvm/Object/ELFObjectFile.h" diff --git a/offload/plugins-nextgen/common/src/JIT.cpp b/offload/plugins-nextgen/common/src/JIT.cpp index 9eb610cab4de66..9d58e6060646ba 100644 --- a/offload/plugins-nextgen/common/src/JIT.cpp +++ b/offload/plugins-nextgen/common/src/JIT.cpp @@ -56,28 +56,6 @@ bool isImageBitcode(const __tgt_device_image &Image) { return identify_magic(Binary) == file_magic::bitcode; } -std::once_flag InitFlag; - -void init(Triple TT) { - codegen::RegisterCodeGenFlags(); -#ifdef LIBOMPTARGET_JIT_NVPTX - if (TT.isNVPTX()) { - LLVMInitializeNVPTXTargetInfo(); - LLVMInitializeNVPTXTarget(); - LLVMInitializeNVPTXTargetMC(); - LLVMInitializeNVPTXAsmPrinter(); - } -#endif -#ifdef LIBOMPTARGET_JIT_AMDGPU - if (TT.isAMDGPU()) { - LLVMInitializeAMDGPUTargetInfo(); - LLVMInitializeAMDGPUTarget(); - LLVMInitializeAMDGPUTargetMC(); - LLVMInitializeAMDGPUAsmPrinter(); - } -#endif -} - Expected> createModuleFromMemoryBuffer(std::unique_ptr &MB, LLVMContext &Context) { @@ -148,7 +126,23 @@ createTargetMachine(Module &M, std::string CPU, unsigned OptLevel) { } // namespace JITEngine::JITEngine(Triple::ArchType TA) : TT(Triple::getArchTypeName(TA)) { - std::call_once(InitFlag, init, TT); + codegen::RegisterCodeGenFlags(); +#ifdef LIBOMPTARGET_JIT_NVPTX + if (TT.isNVPTX()) { + LLVMInitializeNVPTXTargetInfo(); + LLVMInitializeNVPTXTarget(); + LLVMInitializeNVPTXTargetMC(); + LLVMInitializeNVPTXAsmPrinter(); + } +#endif +#ifdef LIBOMPTARGET_JIT_AMDGPU + if (TT.isAMDGPU()) { + LLVMInitializeAMDGPUTargetInfo(); + LLVMInitializeAMDGPUTarget(); + LLVMInitializeAMDGPUTargetMC(); + LLVMInitializeAMDGPUAsmPrinter(); + } +#endif } void JITEngine::opt(TargetMachine *TM, TargetLibraryInfoImpl *TLII, Module &M, diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp index 8de93ba17a560b..fae197527850a6 100644 --- a/offload/plugins-nextgen/common/src/PluginInterface.cpp +++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp @@ -13,7 +13,6 @@ #include "Shared/APITypes.h" #include "Shared/Debug.h" #include "Shared/Environment.h" -#include "Shared/PluginAPI.h" #include "GlobalHandler.h" #include "JIT.h" @@ -39,8 +38,6 @@ using namespace omp; using namespace target; using namespace plugin; -GenericPluginTy *PluginTy::SpecificPlugin = nullptr; - // TODO: Fix any thread safety issues for multi-threaded kernel recording. struct RecordReplayTy { @@ -2035,205 +2032,3 @@ bool llvm::omp::target::plugin::libomptargetSupportsRPC() { return false; #endif } - -/// Exposed library API function, basically wrappers around the GenericDeviceTy -/// functionality with the same name. All non-async functions are redirected -/// to the async versions right away with a NULL AsyncInfoPtr. -#ifdef __cplusplus -extern "C" { -#endif - -int32_t __tgt_rtl_init_plugin() { - auto Err = PluginTy::initIfNeeded(); - if (Err) { - [[maybe_unused]] std::string ErrStr = toString(std::move(Err)); - DP("Failed to init plugin: %s", ErrStr.c_str()); - return OFFLOAD_FAIL; - } - - return OFFLOAD_SUCCESS; -} - -int32_t __tgt_rtl_is_valid_binary(__tgt_device_image *Image) { - if (!PluginTy::isActive()) - return false; - - return PluginTy::get().is_valid_binary(Image); -} - -int32_t __tgt_rtl_init_device(int32_t DeviceId) { - return PluginTy::get().init_device(DeviceId); -} - -int32_t __tgt_rtl_number_of_devices() { - return PluginTy::get().number_of_devices(); -} - -int64_t __tgt_rtl_init_requires(int64_t RequiresFlags) { - return PluginTy::get().init_requires(RequiresFlags); -} - -int32_t __tgt_rtl_is_data_exchangable(int32_t SrcDeviceId, - int32_t DstDeviceId) { - return PluginTy::get().is_data_exchangable(SrcDeviceId, DstDeviceId); -} - -int32_t __tgt_rtl_initialize_record_replay(int32_t DeviceId, int64_t MemorySize, - void *VAddr, bool isRecord, - bool SaveOutput, - uint64_t &ReqPtrArgOffset) { - return PluginTy::get().initialize_record_replay( - DeviceId, MemorySize, VAddr, isRecord, SaveOutput, ReqPtrArgOffset); -} - -int32_t __tgt_rtl_load_binary(int32_t DeviceId, __tgt_device_image *TgtImage, - __tgt_device_binary *Binary) { - return PluginTy::get().load_binary(DeviceId, TgtImage, Binary); -} - -void *__tgt_rtl_data_alloc(int32_t DeviceId, int64_t Size, void *HostPtr, - int32_t Kind) { - return PluginTy::get().data_alloc(DeviceId, Size, HostPtr, Kind); -} - -int32_t __tgt_rtl_data_delete(int32_t DeviceId, void *TgtPtr, int32_t Kind) { - return PluginTy::get().data_delete(DeviceId, TgtPtr, Kind); -} - -int32_t __tgt_rtl_data_lock(int32_t DeviceId, void *Ptr, int64_t Size, - void **LockedPtr) { - return PluginTy::get().data_lock(DeviceId, Ptr, Size, LockedPtr); -} - -int32_t __tgt_rtl_data_unlock(int32_t DeviceId, void *Ptr) { - return PluginTy::get().data_unlock(DeviceId, Ptr); -} - -int32_t __tgt_rtl_data_notify_mapped(int32_t DeviceId, void *HstPtr, - int64_t Size) { - return PluginTy::get().data_notify_mapped(DeviceId, HstPtr, Size); -} - -int32_t __tgt_rtl_data_notify_unmapped(int32_t DeviceId, void *HstPtr) { - return PluginTy::get().data_notify_unmapped(DeviceId, HstPtr); -} - -int32_t __tgt_rtl_data_submit(int32_t DeviceId, void *TgtPtr, void *HstPtr, - int64_t Size) { - return PluginTy::get().data_submit(DeviceId, TgtPtr, HstPtr, Size); -} - -int32_t __tgt_rtl_data_submit_async(int32_t DeviceId, void *TgtPtr, - void *HstPtr, int64_t Size, - __tgt_async_info *AsyncInfoPtr) { - return PluginTy::get().data_submit_async(DeviceId, TgtPtr, HstPtr, Size, - AsyncInfoPtr); -} - -int32_t __tgt_rtl_data_retrieve(int32_t DeviceId, void *HstPtr, void *TgtPtr, - int64_t Size) { - return PluginTy::get().data_retrieve(DeviceId, HstPtr, TgtPtr, Size); -} - -int32_t __tgt_rtl_data_retrieve_async(int32_t DeviceId, void *HstPtr, - void *TgtPtr, int64_t Size, - __tgt_async_info *AsyncInfoPtr) { - return PluginTy::get().data_retrieve_async(DeviceId, HstPtr, TgtPtr, Size, - AsyncInfoPtr); -} - -int32_t __tgt_rtl_data_exchange(int32_t SrcDeviceId, void *SrcPtr, - int32_t DstDeviceId, void *DstPtr, - int64_t Size) { - return PluginTy::get().data_exchange(SrcDeviceId, SrcPtr, DstDeviceId, DstPtr, - Size); -} - -int32_t __tgt_rtl_data_exchange_async(int32_t SrcDeviceId, void *SrcPtr, - int DstDeviceId, void *DstPtr, - int64_t Size, - __tgt_async_info *AsyncInfo) { - return PluginTy::get().data_exchange_async(SrcDeviceId, SrcPtr, DstDeviceId, - DstPtr, Size, AsyncInfo); -} - -int32_t __tgt_rtl_launch_kernel(int32_t DeviceId, void *TgtEntryPtr, - void **TgtArgs, ptrdiff_t *TgtOffsets, - KernelArgsTy *KernelArgs, - __tgt_async_info *AsyncInfoPtr) { - return PluginTy::get().launch_kernel(DeviceId, TgtEntryPtr, TgtArgs, - TgtOffsets, KernelArgs, AsyncInfoPtr); -} - -int32_t __tgt_rtl_synchronize(int32_t DeviceId, - __tgt_async_info *AsyncInfoPtr) { - return PluginTy::get().synchronize(DeviceId, AsyncInfoPtr); -} - -int32_t __tgt_rtl_query_async(int32_t DeviceId, - __tgt_async_info *AsyncInfoPtr) { - return PluginTy::get().query_async(DeviceId, AsyncInfoPtr); -} - -void __tgt_rtl_print_device_info(int32_t DeviceId) { - PluginTy::get().print_device_info(DeviceId); -} - -int32_t __tgt_rtl_create_event(int32_t DeviceId, void **EventPtr) { - return PluginTy::get().create_event(DeviceId, EventPtr); -} - -int32_t __tgt_rtl_record_event(int32_t DeviceId, void *EventPtr, - __tgt_async_info *AsyncInfoPtr) { - return PluginTy::get().record_event(DeviceId, EventPtr, AsyncInfoPtr); -} - -int32_t __tgt_rtl_wait_event(int32_t DeviceId, void *EventPtr, - __tgt_async_info *AsyncInfoPtr) { - return PluginTy::get().wait_event(DeviceId, EventPtr, AsyncInfoPtr); -} - -int32_t __tgt_rtl_sync_event(int32_t DeviceId, void *EventPtr) { - return PluginTy::get().sync_event(DeviceId, EventPtr); -} - -int32_t __tgt_rtl_destroy_event(int32_t DeviceId, void *EventPtr) { - return PluginTy::get().destroy_event(DeviceId, EventPtr); -} - -void __tgt_rtl_set_info_flag(uint32_t NewInfoLevel) { - return PluginTy::get().set_info_flag(NewInfoLevel); -} - -int32_t __tgt_rtl_init_async_info(int32_t DeviceId, - __tgt_async_info **AsyncInfoPtr) { - return PluginTy::get().init_async_info(DeviceId, AsyncInfoPtr); -} - -int32_t __tgt_rtl_init_device_info(int32_t DeviceId, - __tgt_device_info *DeviceInfo, - const char **ErrStr) { - return PluginTy::get().init_device_info(DeviceId, DeviceInfo, ErrStr); -} - -int32_t __tgt_rtl_set_device_offset(int32_t DeviceIdOffset) { - return PluginTy::get().set_device_offset(DeviceIdOffset); -} - -int32_t __tgt_rtl_use_auto_zero_copy(int32_t DeviceId) { - return PluginTy::get().use_auto_zero_copy(DeviceId); -} - -int32_t __tgt_rtl_get_global(__tgt_device_binary Binary, uint64_t Size, - const char *Name, void **DevicePtr) { - return PluginTy::get().get_global(Binary, Size, Name, DevicePtr); -} - -int32_t __tgt_rtl_get_function(__tgt_device_binary Binary, const char *Name, - void **KernelPtr) { - return PluginTy::get().get_function(Binary, Name, KernelPtr); -} - -#ifdef __cplusplus -} -#endif diff --git a/offload/plugins-nextgen/cuda/CMakeLists.txt b/offload/plugins-nextgen/cuda/CMakeLists.txt index 0284bd22d2a4df..dd684bb223431d 100644 --- a/offload/plugins-nextgen/cuda/CMakeLists.txt +++ b/offload/plugins-nextgen/cuda/CMakeLists.txt @@ -51,8 +51,3 @@ else() libomptarget_say("Not generating NVIDIA tests, no supported devices detected." " Use 'LIBOMPTARGET_FORCE_NVIDIA_TESTS' to override.") endif() - -# Install plugin under the lib destination folder. -install(TARGETS omptarget.rtl.cuda LIBRARY DESTINATION "${OFFLOAD_INSTALL_LIBDIR}") -set_target_properties(omptarget.rtl.cuda PROPERTIES - INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..") diff --git a/offload/plugins-nextgen/cuda/src/rtl.cpp b/offload/plugins-nextgen/cuda/src/rtl.cpp index fc74c6aa23fddd..b260334baa18b0 100644 --- a/offload/plugins-nextgen/cuda/src/rtl.cpp +++ b/offload/plugins-nextgen/cuda/src/rtl.cpp @@ -1342,10 +1342,6 @@ struct CUDAPluginTy final : public GenericPluginTy { return 0; } -#ifdef OMPT_SUPPORT - ompt::connectLibrary(); -#endif - if (Res == CUDA_ERROR_NO_DEVICE) { // Do not initialize if there are no devices. DP("There are no devices supporting CUDA.\n"); @@ -1390,6 +1386,8 @@ struct CUDAPluginTy final : public GenericPluginTy { return Triple::nvptx64; } + const char *getName() const override { return GETNAME(TARGET_NAME); } + /// Check whether the image is compatible with the available CUDA devices. Expected isELFCompatible(StringRef Image) const override { auto ElfOrErr = @@ -1495,8 +1493,6 @@ Error CUDADeviceTy::dataExchangeImpl(const void *SrcPtr, return Plugin::check(Res, "Error in cuMemcpyDtoDAsync: %s"); } -GenericPluginTy *PluginTy::createPlugin() { return new CUDAPluginTy(); } - template static Error Plugin::check(int32_t Code, const char *ErrFmt, ArgsTy... Args) { CUresult ResultCode = static_cast(Code); @@ -1516,3 +1512,9 @@ static Error Plugin::check(int32_t Code, const char *ErrFmt, ArgsTy... Args) { } // namespace target } // namespace omp } // namespace llvm + +extern "C" { +llvm::omp::target::plugin::GenericPluginTy *createPlugin_cuda() { + return new llvm::omp::target::plugin::CUDAPluginTy(); +} +} diff --git a/offload/plugins-nextgen/host/CMakeLists.txt b/offload/plugins-nextgen/host/CMakeLists.txt index 1d000442c84d46..72b5681283fe28 100644 --- a/offload/plugins-nextgen/host/CMakeLists.txt +++ b/offload/plugins-nextgen/host/CMakeLists.txt @@ -31,14 +31,6 @@ else() target_include_directories(omptarget.rtl.host PRIVATE dynamic_ffi) endif() -# Install plugin under the lib destination folder. -install(TARGETS omptarget.rtl.host - LIBRARY DESTINATION "${OFFLOAD_INSTALL_LIBDIR}") -set_target_properties(omptarget.rtl.host PROPERTIES - INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/.." - POSITION_INDEPENDENT_CODE ON - CXX_VISIBILITY_PRESET protected) - target_include_directories(omptarget.rtl.host PRIVATE ${LIBOMPTARGET_INCLUDE_DIR}) diff --git a/offload/plugins-nextgen/host/src/rtl.cpp b/offload/plugins-nextgen/host/src/rtl.cpp index 4bdcae3dd6a1b5..409b44b1640ad7 100644 --- a/offload/plugins-nextgen/host/src/rtl.cpp +++ b/offload/plugins-nextgen/host/src/rtl.cpp @@ -385,10 +385,6 @@ struct GenELF64PluginTy final : public GenericPluginTy { /// Initialize the plugin and return the number of devices. Expected initImpl() override { -#ifdef OMPT_SUPPORT - ompt::connectLibrary(); -#endif - #ifdef USES_DYNAMIC_FFI if (auto Err = Plugin::check(ffi_init(), "Failed to initialize libffi")) return std::move(Err); @@ -445,9 +441,9 @@ struct GenELF64PluginTy final : public GenericPluginTy { return llvm::Triple::UnknownArch; #endif } -}; -GenericPluginTy *PluginTy::createPlugin() { return new GenELF64PluginTy(); } + const char *getName() const override { return GETNAME(TARGET_NAME); } +}; template static Error Plugin::check(int32_t Code, const char *ErrMsg, ArgsTy... Args) { @@ -462,3 +458,9 @@ static Error Plugin::check(int32_t Code, const char *ErrMsg, ArgsTy... Args) { } // namespace target } // namespace omp } // namespace llvm + +extern "C" { +llvm::omp::target::plugin::GenericPluginTy *createPlugin_host() { + return new llvm::omp::target::plugin::GenELF64PluginTy(); +} +} diff --git a/offload/src/CMakeLists.txt b/offload/src/CMakeLists.txt index eda5a85ff1ab80..8fe6d19d83eb36 100644 --- a/offload/src/CMakeLists.txt +++ b/offload/src/CMakeLists.txt @@ -65,6 +65,10 @@ target_compile_definitions(omptarget PRIVATE DEBUG_PREFIX="omptarget" ) +foreach(plugin IN LISTS LIBOMPTARGET_PLUGINS_TO_BUILD) + target_link_libraries(omptarget PRIVATE omptarget.rtl.${plugin}) +endforeach() + target_compile_options(omptarget PUBLIC ${offload_compile_flags}) target_link_options(omptarget PUBLIC ${offload_link_flags}) diff --git a/offload/src/OffloadRTL.cpp b/offload/src/OffloadRTL.cpp index dd75b1b181505e..29b573a27d087d 100644 --- a/offload/src/OffloadRTL.cpp +++ b/offload/src/OffloadRTL.cpp @@ -50,6 +50,7 @@ void deinitRuntime() { if (RefCount == 1) { DP("Deinit offload library!\n"); + PM->deinit(); delete PM; PM = nullptr; } diff --git a/offload/src/OpenMP/InteropAPI.cpp b/offload/src/OpenMP/InteropAPI.cpp index 1a995cde7816e1..bdbc440c64a2c9 100644 --- a/offload/src/OpenMP/InteropAPI.cpp +++ b/offload/src/OpenMP/InteropAPI.cpp @@ -230,14 +230,14 @@ void __tgt_interop_init(ident_t *LocRef, int32_t Gtid, } DeviceTy &Device = *DeviceOrErr; - if (!Device.RTL || !Device.RTL->init_device_info || + if (!Device.RTL || Device.RTL->init_device_info(DeviceId, &(InteropPtr)->device_info, &(InteropPtr)->err_str)) { delete InteropPtr; InteropPtr = omp_interop_none; } if (InteropType == kmp_interop_type_tasksync) { - if (!Device.RTL || !Device.RTL->init_async_info || + if (!Device.RTL || Device.RTL->init_async_info(DeviceId, &(InteropPtr)->async_info)) { delete InteropPtr; InteropPtr = omp_interop_none; diff --git a/offload/src/PluginManager.cpp b/offload/src/PluginManager.cpp index dbb556c179e58c..191afa345641a1 100644 --- a/offload/src/PluginManager.cpp +++ b/offload/src/PluginManager.cpp @@ -23,85 +23,25 @@ using namespace llvm::sys; PluginManager *PM = nullptr; -Expected> -PluginAdaptorTy::create(const std::string &Name) { - DP("Attempting to load library '%s'...\n", Name.c_str()); - TIMESCOPE_WITH_NAME_AND_IDENT(Name, (const ident_t *)nullptr); - - std::string ErrMsg; - auto LibraryHandler = std::make_unique( - DynamicLibrary::getPermanentLibrary(Name.c_str(), &ErrMsg)); - - if (!LibraryHandler->isValid()) { - // Library does not exist or cannot be found. - return createStringError(inconvertibleErrorCode(), - "Unable to load library '%s': %s!\n", Name.c_str(), - ErrMsg.c_str()); - } - - DP("Successfully loaded library '%s'!\n", Name.c_str()); - auto PluginAdaptor = std::unique_ptr( - new PluginAdaptorTy(Name, std::move(LibraryHandler))); - if (auto Err = PluginAdaptor->init()) - return Err; - return std::move(PluginAdaptor); -} - -PluginAdaptorTy::PluginAdaptorTy(const std::string &Name, - std::unique_ptr DL) - : Name(Name), LibraryHandler(std::move(DL)) {} - -Error PluginAdaptorTy::init() { - -#define PLUGIN_API_HANDLE(NAME) \ - NAME = reinterpret_cast( \ - LibraryHandler->getAddressOfSymbol(GETNAME(__tgt_rtl_##NAME))); \ - if (!NAME) { \ - return createStringError(inconvertibleErrorCode(), \ - "Invalid plugin as necessary interface function " \ - "(%s) was not found.\n", \ - std::string(#NAME).c_str()); \ - } - -#include "Shared/PluginAPI.inc" -#undef PLUGIN_API_HANDLE - - // Remove plugin on failure to call optional init_plugin - int32_t Rc = init_plugin(); - if (Rc != OFFLOAD_SUCCESS) { - return createStringError(inconvertibleErrorCode(), - "Unable to initialize library '%s': %u!\n", - Name.c_str(), Rc); - } - - // No devices are supported by this RTL? - int32_t NumberOfPluginDevices = number_of_devices(); - if (!NumberOfPluginDevices) { - return createStringError(inconvertibleErrorCode(), - "No devices supported in this RTL\n"); - } - - DP("Registered '%s' with %d plugin visible devices!\n", Name.c_str(), - NumberOfPluginDevices); - return Error::success(); -} +// Every plugin exports this method to create an instance of the plugin type. +#define PLUGIN_TARGET(Name) extern "C" GenericPluginTy *createPlugin_##Name(); +#include "Shared/Targets.def" void PluginManager::init() { TIMESCOPE(); DP("Loading RTLs...\n"); - // Attempt to open all the plugins and, if they exist, check if the interface - // is correct and if they are supporting any devices. + // Attempt to create an instance of each supported plugin. #define PLUGIN_TARGET(Name) \ do { \ - auto PluginAdaptorOrErr = \ - PluginAdaptorTy::create("libomptarget.rtl." #Name ".so"); \ - if (!PluginAdaptorOrErr) { \ - [[maybe_unused]] std::string InfoMsg = \ - toString(PluginAdaptorOrErr.takeError()); \ - DP("%s", InfoMsg.c_str()); \ + auto Plugin = std::unique_ptr(createPlugin_##Name()); \ + if (auto Err = Plugin->init()) { \ + [[maybe_unused]] std::string InfoMsg = toString(std::move(Err)); \ + DP("Failed to init plugin: %s\n", InfoMsg.c_str()); \ } else { \ - PluginAdaptors.push_back(std::move(*PluginAdaptorOrErr)); \ + DP("Registered plugin %s with %d visible device(s)\n", \ + Plugin->getName(), Plugin->number_of_devices()); \ + Plugins.emplace_back(std::move(Plugin)); \ } \ } while (false); #include "Shared/Targets.def" @@ -109,15 +49,29 @@ void PluginManager::init() { DP("RTLs loaded!\n"); } -void PluginManager::initDevices(PluginAdaptorTy &RTL) { +void PluginManager::deinit() { + TIMESCOPE(); + DP("Unloading RTLs...\n"); + + for (auto &Plugin : Plugins) { + if (auto Err = Plugin->deinit()) { + [[maybe_unused]] std::string InfoMsg = toString(std::move(Err)); + DP("Failed to deinit plugin: %s\n", InfoMsg.c_str()); + } + Plugin.release(); + } + + DP("RTLs unloaded!\n"); +} + +void PluginManager::initDevices(GenericPluginTy &RTL) { // If this RTL has already been initialized. if (PM->DeviceOffsets.contains(&RTL)) return; TIMESCOPE(); // If this RTL is not already in use, initialize it. - assert(RTL.number_of_devices() > 0 && - "Tried to initialize useless plugin adaptor"); + assert(RTL.number_of_devices() > 0 && "Tried to initialize useless plugin!"); // Initialize the device information for the RTL we are about to use. auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor(); @@ -157,13 +111,12 @@ void PluginManager::initDevices(PluginAdaptorTy &RTL) { DeviceOffsets[&RTL] = DeviceOffset; DeviceUsed[&RTL] = NumberOfUserDevices; - DP("Plugin adaptor " DPxMOD " has index %d, exposes %d out of %d devices!\n", - DPxPTR(RTL.LibraryHandler.get()), DeviceOffset, NumberOfUserDevices, - RTL.number_of_devices()); + DP("Plugin has index %d, exposes %d out of %d devices!\n", DeviceOffset, + NumberOfUserDevices, RTL.number_of_devices()); } void PluginManager::initAllPlugins() { - for (auto &R : PluginAdaptors) + for (auto &R : Plugins) initDevices(*R); } @@ -216,19 +169,22 @@ void PluginManager::registerLib(__tgt_bin_desc *Desc) { // Obtain the image and information that was previously extracted. __tgt_device_image *Img = &DI.getExecutableImage(); - PluginAdaptorTy *FoundRTL = nullptr; + GenericPluginTy *FoundRTL = nullptr; // Scan the RTLs that have associated images until we find one that supports // the current image. - for (auto &R : PM->pluginAdaptors()) { + for (auto &R : PM->plugins()) { + if (!R.number_of_devices()) + continue; + if (!R.is_valid_binary(Img)) { DP("Image " DPxMOD " is NOT compatible with RTL %s!\n", - DPxPTR(Img->ImageStart), R.Name.c_str()); + DPxPTR(Img->ImageStart), R.getName()); continue; } DP("Image " DPxMOD " is compatible with RTL %s!\n", - DPxPTR(Img->ImageStart), R.Name.c_str()); + DPxPTR(Img->ImageStart), R.getName()); PM->initDevices(R); @@ -247,7 +203,7 @@ void PluginManager::registerLib(__tgt_bin_desc *Desc) { (PM->HostEntriesBeginToTransTable)[Desc->HostEntriesBegin]; DP("Registering image " DPxMOD " with RTL %s!\n", DPxPTR(Img->ImageStart), - R.Name.c_str()); + R.getName()); registerImageIntoTranslationTable(TransTable, PM->DeviceOffsets[&R], PM->DeviceUsed[&R], Img); @@ -282,11 +238,11 @@ void PluginManager::unregisterLib(__tgt_bin_desc *Desc) { // Obtain the image and information that was previously extracted. __tgt_device_image *Img = &DI.getExecutableImage(); - PluginAdaptorTy *FoundRTL = NULL; + GenericPluginTy *FoundRTL = NULL; // Scan the RTLs that have associated images until we find one that supports // the current image. We only need to scan RTLs that are already being used. - for (auto &R : PM->pluginAdaptors()) { + for (auto &R : PM->plugins()) { if (!DeviceOffsets.contains(&R)) continue; @@ -296,8 +252,7 @@ void PluginManager::unregisterLib(__tgt_bin_desc *Desc) { FoundRTL = &R; - DP("Unregistered image " DPxMOD " from RTL " DPxMOD "!\n", - DPxPTR(Img->ImageStart), DPxPTR(R.LibraryHandler.get())); + DP("Unregistered image " DPxMOD " from RTL\n", DPxPTR(Img->ImageStart)); break; } diff --git a/offload/src/device.cpp b/offload/src/device.cpp index 44a2facc8d3ddd..749b4c567f8e46 100644 --- a/offload/src/device.cpp +++ b/offload/src/device.cpp @@ -64,7 +64,7 @@ int HostDataToTargetTy::addEventIfNecessary(DeviceTy &Device, return OFFLOAD_SUCCESS; } -DeviceTy::DeviceTy(PluginAdaptorTy *RTL, int32_t DeviceID, int32_t RTLDeviceID) +DeviceTy::DeviceTy(GenericPluginTy *RTL, int32_t DeviceID, int32_t RTLDeviceID) : DeviceID(DeviceID), RTL(RTL), RTLDeviceID(RTLDeviceID), MappingInfo(*this) {} @@ -192,7 +192,6 @@ int32_t DeviceTy::dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr, RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr, Size, /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);) if (!AsyncInfo) { - assert(RTL->data_exchange && "RTL->data_exchange is nullptr"); return RTL->data_exchange(RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr, Size); } diff --git a/offload/src/interface.cpp b/offload/src/interface.cpp index 557703632c6258..763b051cc6d771 100644 --- a/offload/src/interface.cpp +++ b/offload/src/interface.cpp @@ -456,8 +456,6 @@ EXTERN void __tgt_set_info_flag(uint32_t NewInfoLevel) { assert(PM && "Runtime not initialized"); std::atomic &InfoLevel = getInfoLevelInternal(); InfoLevel.store(NewInfoLevel); - for (auto &R : PM->pluginAdaptors()) - R.set_info_flag(NewInfoLevel); } EXTERN int __tgt_print_device_info(int64_t DeviceId) { diff --git a/offload/tools/kernelreplay/llvm-omp-kernel-replay.cpp b/offload/tools/kernelreplay/llvm-omp-kernel-replay.cpp index 761e04e4c7bbdb..1e9a6a84d80583 100644 --- a/offload/tools/kernelreplay/llvm-omp-kernel-replay.cpp +++ b/offload/tools/kernelreplay/llvm-omp-kernel-replay.cpp @@ -13,8 +13,6 @@ #include "omptarget.h" -#include "Shared/PluginAPI.h" - #include "llvm/Support/CommandLine.h" #include "llvm/Support/JSON.h" #include "llvm/Support/MemoryBuffer.h" diff --git a/offload/unittests/Plugins/NextgenPluginsTest.cpp b/offload/unittests/Plugins/NextgenPluginsTest.cpp index 635bd1637c9032..479b3f614aed2b 100644 --- a/offload/unittests/Plugins/NextgenPluginsTest.cpp +++ b/offload/unittests/Plugins/NextgenPluginsTest.cpp @@ -6,7 +6,6 @@ // //===----------------------------------------------------------------------===// -#include "Shared/PluginAPI.h" #include "omptarget.h" #include "gtest/gtest.h"