[SYCL] Add more cleanup points close to the shutdown (intel#11119)

Previously we tried to release internal resources only when the last
(likely main) thread exits. For multi threaded applications it could be
not functional. e.g. threads are closed during module unload and when
main thread exits - other threads are still alive. Adding logic to try
non blocking release for every thread exit and blocking for the last
one.

---------

Signed-off-by: Tikhomirova, Kseniya <kseniya.tikhomirova@intel.com>

sycl/source/detail/global_handler.cpp

@@ -52,13 +52,11 @@ class ObjectUsageCounter {
     if (!MModifyCounter)
       return;
 
+    LockGuard Guard(GlobalHandler::MSyclGlobalHandlerProtector);
     MCounter--;
-    if (!MCounter) {
-      LockGuard Guard(GlobalHandler::MSyclGlobalHandlerProtector);
-      GlobalHandler *RTGlobalObjHandler = GlobalHandler::getInstancePtr();
-      if (RTGlobalObjHandler) {
-        RTGlobalObjHandler->prepareSchedulerToRelease();
-      }
+    GlobalHandler *RTGlobalObjHandler = GlobalHandler::getInstancePtr();
+    if (RTGlobalObjHandler) {
+      RTGlobalObjHandler->prepareSchedulerToRelease(!MCounter);
     }
   }
 
@@ -142,7 +140,7 @@ void GlobalHandler::attachScheduler(Scheduler *Scheduler) {
   // The method is used in unit tests only. Do not protect with lock since
   // releaseResources will cause dead lock due to host queue release
   if (MScheduler.Inst)
-    prepareSchedulerToRelease();
+    prepareSchedulerToRelease(true);
   MScheduler.Inst.reset(Scheduler);
 }
 
@@ -273,11 +271,13 @@ void GlobalHandler::unloadPlugins() {
   getPlugins().clear();
 }
 
-void GlobalHandler::prepareSchedulerToRelease() {
+void GlobalHandler::prepareSchedulerToRelease(bool Blocking) {
 #ifndef _WIN32
-  drainThreadPool();
+  if (Blocking)
+    drainThreadPool();
   if (MScheduler.Inst)
-    MScheduler.Inst->releaseResources();
+    MScheduler.Inst->releaseResources(Blocking ? BlockingT::BLOCKING
+                                               : BlockingT::NON_BLOCKING);
 #endif
 }
 
@@ -306,7 +306,7 @@ void shutdown() {
 
   // Ensure neither host task is working so that no default context is accessed
   // upon its release
-  Handler->prepareSchedulerToRelease();
+  Handler->prepareSchedulerToRelease(true);
 
   if (Handler->MHostTaskThreadPool.Inst)
     Handler->MHostTaskThreadPool.Inst->finishAndWait();

sycl/source/detail/global_handler.hpp

@@ -80,7 +80,7 @@ class GlobalHandler {
   void unloadPlugins();
   void releaseDefaultContexts();
   void drainThreadPool();
-  void prepareSchedulerToRelease();
+  void prepareSchedulerToRelease(bool Blocking);
 
   void InitXPTI();
   void TraceEventXPTI(const char *Message);

sycl/source/detail/scheduler/scheduler.cpp

@@ -405,13 +405,13 @@ Scheduler::Scheduler() {
 
 Scheduler::~Scheduler() { DefaultHostQueue.reset(); }
 
-void Scheduler::releaseResources() {
+void Scheduler::releaseResources(BlockingT Blocking) {
   //  There might be some commands scheduled for post enqueue cleanup that
   //  haven't been freed because of the graph mutex being locked at the time,
   //  clean them up now.
   cleanupCommands({});
 
-  cleanupAuxiliaryResources(BlockingT::BLOCKING);
+  cleanupAuxiliaryResources(Blocking);
   // We need loop since sometimes we may need new objects to be added to
   // deferred mem objects storage during cleanup. Known example is: we cleanup
   // existing deferred mem objects under write lock, during this process we
@@ -420,7 +420,7 @@ void Scheduler::releaseResources() {
   // with size only so all confitions for deferred release are satisfied) is
   // added to deferred mem obj storage. So we may end up with leak.
   while (!isDeferredMemObjectsEmpty())
-    cleanupDeferredMemObjects(BlockingT::BLOCKING);
+    cleanupDeferredMemObjects(Blocking);
 }
 
 MemObjRecord *Scheduler::getMemObjRecord(const Requirement *const Req) {

sycl/source/detail/scheduler/scheduler.hpp

@@ -461,7 +461,7 @@ class Scheduler {
 
   Scheduler();
   ~Scheduler();
-  void releaseResources();
+  void releaseResources(BlockingT Blocking = BlockingT::BLOCKING);
   bool isDeferredMemObjectsEmpty();
 
   void enqueueCommandForCG(EventImplPtr NewEvent,