Implement multiplexing strategy

CMakeLists.txt

@@ -65,7 +65,9 @@ endforeach()
 
 # =============================================================================
 # CHIP-SPV CMAKE DEPENDENCIES
-if(NOT DEFINED OpenCL_LIBRARY)
+option(CHIP_ENABLE_OPENCL "Enable building the OpenCL backend" ON)
+option(CHIP_ENABLE_LEVEL0 "Enable building the Level Zero backend" ON)
+if(NOT DEFINED OpenCL_LIBRARY AND CHIP_ENABLE_OPENCL)
  message(STATUS "OpenCL_LIBRARY was not set. Searching for libOpenCL.so in LD_LIBRARY_PATH")
  find_library(OpenCL_LIBRARY NAMES OpenCL PATHS ENV LD_LIBRARY_PATH ./ NO_CACHE)
  if(OpenCL_LIBRARY)
@@ -76,7 +78,7 @@ if(NOT DEFINED OpenCL_LIBRARY)
  endif()
 endif()
 
-if(NOT DEFINED LevelZero_LIBRARY)
+if(NOT DEFINED LevelZero_LIBRARY AND CHIP_ENABLE_LEVEL0)
  message(STATUS "LevelZero_LIBRARY was not set. Searching for ze_loader.so in LD_LIBRARY_PATH")
  find_library(LevelZero_LIBRARY NAMES ze_loader PATHS ENV LD_LIBRARY_PATH ./ NO_CACHE)
  if(LevelZero_LIBRARY)
@@ -87,10 +89,14 @@ if(NOT DEFINED LevelZero_LIBRARY)
  endif()
 endif()
 
-message(STATUS "OpenCL_LIBRARY: ${OpenCL_LIBRARY}")
+if(CHIP_ENABLE_OPENCL)
+ message(STATUS "OpenCL_LIBRARY: ${OpenCL_LIBRARY}")
+endif()
+if(CHIP_ENABLE_LEVEL0)
 message(STATUS "LevelZero_LIBRARY: ${LevelZero_LIBRARY}")
+endif()
 
-if(NOT OpenCL_LIBRARY AND NOT LevelZero_LIBRARY)
+if(NOT (OpenCL_LIBRARY AND CHIP_ENABLE_OPENCL) AND NOT (LevelZero_LIBRARY AND CHIP_ENABLE_LEVEL0))
  message(FATAL_ERROR "At least one of OpenCL,Level0 libraries must be available")
 endif()
 
@@ -128,13 +134,13 @@ set(CHIP_SRC
  src/SPIRVFuncInfo.cc
 )
 
-if(OpenCL_LIBRARY)
+if(OpenCL_LIBRARY AND CHIP_ENABLE_OPENCL)
  list(APPEND CHIP_SRC
  src/backend/OpenCL/CHIPBackendOpenCL.cc
  src/backend/OpenCL/SVMemoryRegion.cc)
 endif()
 
-if(LevelZero_LIBRARY)
+if(LevelZero_LIBRARY AND CHIP_ENABLE_LEVEL0)
  list(APPEND CHIP_SRC
  src/backend/Level0/CHIPBackendLevel0.cc)
 endif()
@@ -295,12 +301,12 @@ endif()
 
 set(CHIP_INTERFACE_LIBS ${PTHREAD_LIBRARY})
 
-if(OpenCL_LIBRARY)
+if(OpenCL_LIBRARY AND CHIP_ENABLE_OPENCL)
  list(APPEND CHIP_SPV_DEFINITIONS HAVE_OPENCL)
  list(PREPEND CHIP_INTERFACE_LIBS ${OpenCL_LIBRARY})
 endif()
 
-if(LevelZero_LIBRARY)
+if(LevelZero_LIBRARY AND CHIP_ENABLE_LEVEL0)
  list(APPEND CHIP_SPV_DEFINITIONS HAVE_LEVEL0)
  list(PREPEND CHIP_INTERFACE_LIBS ${LevelZero_LIBRARY})
 endif()
@@ -422,13 +428,13 @@ set(HIP_OFFLOAD_COMPILE_OPTIONS_BUILD_
 list(APPEND HIP_OFFLOAD_LINK_OPTIONS_INSTALL_ "-L${LIB_INSTALL_DIR}" "-lCHIP")
 list(APPEND HIP_OFFLOAD_LINK_OPTIONS_BUILD_ "-L${CMAKE_BINARY_DIR}" "-lCHIP")
 
-if(OpenCL_LIBRARY)
+if(OpenCL_LIBRARY AND CHIP_ENABLE_OPENCL)
  target_link_options(CHIP PUBLIC -Wl,-rpath,${OpenCL_DIR})
  target_link_directories(CHIP PUBLIC ${OpenCL_DIR})
  target_link_libraries(CHIP PUBLIC OpenCL)
 endif()
 
-if(LevelZero_LIBRARY)
+if(LevelZero_LIBRARY AND CHIP_ENABLE_LEVEL0)
  target_link_options(CHIP PUBLIC -Wl,-rpath,${LevelZero_DIR})
  target_link_directories(CHIP PUBLIC ${LevelZero_DIR})
  target_link_libraries(CHIP PUBLIC ze_loader)
@@ -673,10 +679,10 @@ endif()
 
 # Short Summary 
 # print if Level Zero or OpenCL are enabbled
-if(OpenCL_LIBRARY)
+if(OpenCL_LIBRARY AND CHIP_ENABLE_OPENCL)
  message(STATUS "OpenCL is enabled: ${OpenCL_LIBRARY}")
 endif()
 
-if(LevelZero_LIBRARY)
+if(LevelZero_LIBRARY AND CHIP_ENABLE_LEVEL0)
  message(STATUS "Level Zero is enabled: ${LevelZero_LIBRARY}")
 endif()

src/CHIPBackend.cc

@@ -91,7 +91,7 @@ chipstar::CallbackData::CallbackData(hipStreamCallback_t TheCallbackF,
  CallbackF(TheCallbackF) {}
 
 void chipstar::CallbackData::execute(hipError_t ResultFromDependency) {
- CallbackF(ChipQueue, ResultFromDependency, CallbackArgs);
+ CallbackF(STREAM(ChipQueue), ResultFromDependency, CallbackArgs);
 }
 
 // DeviceVar
@@ -480,9 +480,9 @@ void chipstar::ExecItem::copyArgs(void **Args) {
 }
 
 chipstar::ExecItem::ExecItem(dim3 GridDim, dim3 BlockDim, size_t SharedMem,
- hipStream_t ChipQueue)
+ chipstar::Queue *ChipQueue)
  : SharedMem_(SharedMem), GridDim_(GridDim), BlockDim_(BlockDim),
- ChipQueue_(static_cast<chipstar::Queue *>(ChipQueue)){};
+ ChipQueue_(ChipQueue){};
 
 dim3 chipstar::ExecItem::getBlock() { return BlockDim_; }
 dim3 chipstar::ExecItem::getGrid() { return GridDim_; }
@@ -502,11 +502,13 @@ chipstar::Device::~Device() {
  LOCK(DeviceMtx); // chipstar::Device::ChipQueues_
  logDebug("~Device() {}", (void *)this);
  while (this->ChipQueues_.size() > 0) {
- delete ChipQueues_[0];
+ ChipQueues_[0]->~Queue();
+ free(CHIP_OBJ_TO_HANDLE(ChipQueues_[0], ihipStream_t));
  ChipQueues_.erase(ChipQueues_.begin());
  }
 
- delete LegacyDefaultQueue;
+ LegacyDefaultQueue->~Queue();
+ free(CHIP_OBJ_TO_HANDLE(LegacyDefaultQueue, ihipStream_t));
  LegacyDefaultQueue = nullptr;
 }
 chipstar::Queue *chipstar::Device::getLegacyDefaultQueue() {
@@ -540,11 +542,20 @@ chipstar::Queue *chipstar::Device::getPerThreadDefaultQueue() {
  return getPerThreadDefaultQueueNoLock();
 }
 
+void chipstar::Device::QueueDeleter::operator()(
+ chipstar::Queue *q) const noexcept {
+ if (q) {
+ q->~Queue();
+ free(CHIP_OBJ_TO_HANDLE(q, ihipStream_t));
+ }
+}
+
 chipstar::Queue *chipstar::Device::getPerThreadDefaultQueueNoLock() {
  if (!PerThreadDefaultQueue.get()) {
  logDebug("PerThreadDefaultQueue is null.. Creating a new queue.");
  PerThreadDefaultQueue =
- std::unique_ptr<chipstar::Queue>(::Backend->createCHIPQueue(this));
+ std::unique_ptr<chipstar::Queue, chipstar::Device::QueueDeleter>(
+ ::Backend->createCHIPQueue(this), chipstar::Device::QueueDeleter());
  PerThreadStreamUsed_ = true;
  PerThreadDefaultQueue.get()->PerThreadQueueForDevice = this;
  }
@@ -799,7 +810,8 @@ void chipstar::Device::eraseModule(chipstar::Module *Module) {
  LOCK(DeviceMtx); // SrcModToCompiledMod_
  for (auto &Kv : SrcModToCompiledMod_)
  if (Kv.second == Module) {
- delete Module;
+ Module->~Module();
+ free(CHIP_OBJ_TO_HANDLE(Module, ihipModule_t));
  SrcModToCompiledMod_.erase(Kv.first);
  break;
  }
@@ -897,7 +909,8 @@ bool chipstar::Device::removeQueue(chipstar::Queue *ChipQueue) {
  }
  ChipQueues_.erase(FoundQueue);
 
- delete ChipQueue;
+ ChipQueue->~Queue();
+ free(CHIP_OBJ_TO_HANDLE(ChipQueue, ihipStream_t));
  return true;
 }
 
@@ -1228,7 +1241,8 @@ chipstar::Backend::~Backend() {
  UserEvents.clear();
  for (auto &Ctx : ChipContexts) {
  ::Backend->removeContext(Ctx);
- delete Ctx;
+ Ctx->~Context();
+ free(CHIP_OBJ_TO_HANDLE(Ctx, ihipCtx_t));
  }
 }
 
@@ -1364,7 +1378,7 @@ void chipstar::Backend::addContext(chipstar::Context *ChipContext) {
 
 hipError_t chipstar::Backend::configureCall(dim3 Grid, dim3 Block,
  size_t SharedMem,
- hipStream_t ChipQueue) {
+ chipstar::Queue *ChipQueue) {
  logDebug("Backend->configureCall(grid=({},{},{}), block=({},{},{}), "
  "shared={}, q={}",
  Grid.x, Grid.y, Grid.z, Block.x, Block.y, Block.z, SharedMem,
@@ -1483,9 +1497,9 @@ chipstar::Queue *chipstar::Backend::findQueue(chipstar::Queue *ChipQueue) {
  auto Dev = ::Backend->getActiveDevice();
  LOCK(Dev->DeviceMtx); // chipstar::Device::ChipQueues_ via getQueuesNoLock()
 
- if (ChipQueue == hipStreamPerThread) {
+ if (ChipQueue == (chipstar::Queue *)hipStreamPerThread) {
  return Dev->getPerThreadDefaultQueueNoLock();
- } else if (ChipQueue == hipStreamLegacy) {
+ } else if (ChipQueue == (chipstar::Queue *)hipStreamLegacy) {
  return Dev->getLegacyDefaultQueue();
  } else if (ChipQueue == nullptr) {
  return Dev->getDefaultQueue();
@@ -1688,7 +1702,12 @@ void chipstar::Queue::updateLastNode(CHIPGraphNode *NewNode) {
  LastNode_ = NewNode;
 }
 
-void chipstar::Queue::initCaptureGraph() { CaptureGraph_ = new CHIPGraph(); }
+void chipstar::Queue::initCaptureGraph() {
+ void *mem = malloc(sizeof(ihipDispatch) + sizeof(CHIPGraph));
+ CHIPGraph *G = CHIP_HANDLE_TO_OBJ(mem, CHIPGraph);
+ G = new (G) CHIPGraph();
+ CaptureGraph_ = G;
+}
 
 std::shared_ptr<chipstar::Event>
 chipstar::Queue::RegisteredVarCopy(chipstar::ExecItem *ExecItem,
@@ -1883,4 +1902,4 @@ void chipstar::Queue::addCallback(hipStreamCallback_t Callback,
 
 CHIPGraph *chipstar::Queue::getCaptureGraph() const {
  return static_cast<CHIPGraph *>(CaptureGraph_);
-}
+}

src/CHIPBackend.hh

@@ -626,7 +626,7 @@ public:
  void markHasInitializer(bool State = true) { HasInitializer_ = State; }
 };
 
-class Event : public ihipEvent_t {
+class Event {
 protected:
  bool TrackCalled_ = false;
  bool UserEvent_ = false;
@@ -654,6 +654,10 @@ protected:
  virtual ~Event(){};
 
 public:
+ static void deleter(chipstar::Event *e) {
+ e->~Event();
+ free(CHIP_OBJ_TO_HANDLE(e, ihipEvent_t));
+ }
  void markTracked() { TrackCalled_ = true; }
  bool isTrackCalled() { return TrackCalled_; }
  void setTrackCalled(bool Val) { TrackCalled_ = Val; }
@@ -846,7 +850,7 @@ public:
  * ROCclr - amd::Program
  * CUDA - CUmodule
  */
-class Module : public ihipModule_t {
+class Module {
  /// Flag for the allocation state of the device variables. True if
  /// all variables have space allocated for this module for the
  /// device this module is attached to. False implies that
@@ -1006,7 +1010,7 @@ public:
 /**
  * @brief Contains information about the function on the host and device
  */
-class Kernel : public ihipModuleSymbol_t {
+class Kernel {
 protected:
  /**
  * @brief hidden default constructor. Only derived type constructor should be
@@ -1180,7 +1184,7 @@ public:
  * @param chip_queue_
  */
  ExecItem(dim3 GirdDim, dim3 BlockDim, size_t SharedMem,
- hipStream_t ChipQueue);
+ chipstar::Queue *ChipQueue);
 
  /**
  * @brief Set the chipstar::Kernel object
@@ -1290,7 +1294,11 @@ public:
  std::vector<chipstar::Queue *> getQueuesNoLock() { return ChipQueues_; }
 
  chipstar::Queue *LegacyDefaultQueue;
- inline static thread_local std::unique_ptr<chipstar::Queue>
+ struct QueueDeleter {
+ void operator()(chipstar::Queue *q) const noexcept;
+ };
+
+ inline static thread_local std::unique_ptr<chipstar::Queue, QueueDeleter>
  PerThreadDefaultQueue;
 
  /**
@@ -1611,7 +1619,7 @@ protected:
  * multiple devices. Provides for creation of additional queues, events, and
  * interaction with devices.
  */
-class Context : public ihipCtx_t {
+class Context {
 protected:
  int RefCount_;
  chipstar::Device *ChipDevice_;
@@ -1802,7 +1810,7 @@ public:
 
  virtual chipstar::ExecItem *createExecItem(dim3 GirdDim, dim3 BlockDim,
  size_t SharedMem,
- hipStream_t ChipQueue) = 0;
+ chipstar::Queue *ChipQueue) = 0;
 
  int getPerThreadQueuesActive();
  std::mutex SetActiveMtx;
@@ -1956,7 +1964,7 @@ public:
  * @return hipError_t
  */
  hipError_t configureCall(dim3 GridDim, dim3 BlockDim, size_t SharedMem,
- hipStream_t ChipQueue);
+ chipstar::Queue *ChipQueue);
 
  /**
  * @brief Return a device which meets or exceeds the requirements
@@ -2029,11 +2037,11 @@ public:
 /**
  * @brief Queue class for submitting kernels to for execution
  */
-class Queue : public ihipStream_t {
+class Queue {
 protected:
  hipStreamCaptureStatus CaptureStatus_ = hipStreamCaptureStatusNone;
  hipStreamCaptureMode CaptureMode_ = hipStreamCaptureModeGlobal;
- hipGraph_t CaptureGraph_;
+ CHIPGraph *CaptureGraph_;
  std::mutex LastEventMtx;
  /// @brief node for creating a dependency chain between subsequent record
  /// events when in graph capture mode