From 3f7dc12183f731ecbe8cbf3d8775e879f9227b01 Mon Sep 17 00:00:00 2001
From: birchkwok <birch.kwok@tvbc.com.cn>
Date: Mon, 16 Sep 2024 00:33:10 +0800
Subject: [PATCH] Fixed mmap-related errors on Windows.

---
 lynse/execution_layer/indexer.py |  3 +--
 lynse/storage_layer/storage.py   | 24 +++++++++++++++---------
 lynse/utils/utils.py             | 14 ++++----------
 3 files changed, 20 insertions(+), 21 deletions(-)

diff --git a/lynse/execution_layer/indexer.py b/lynse/execution_layer/indexer.py
index 737c00c..bbc8349 100644
--- a/lynse/execution_layer/indexer.py
+++ b/lynse/execution_layer/indexer.py
@@ -166,8 +166,7 @@ def build_binary():
                                                     f'{self.storage_worker.fingerprint}.*SQ8.index')
                     )
 
-                sq8_data = np.load(self.index_data_path / f'{self.storage_worker.fingerprint}.sqd',
-                                   mmap_mode='r')
+                sq8_data = np.load(self.index_data_path / f'{self.storage_worker.fingerprint}.sqd')
 
                 _index.data = binary_data
                 _index.ids = binary_ids
diff --git a/lynse/storage_layer/storage.py b/lynse/storage_layer/storage.py
index 1a154cd..9b7c462 100644
--- a/lynse/storage_layer/storage.py
+++ b/lynse/storage_layer/storage.py
@@ -262,16 +262,22 @@ def return_if_in_memory(self, filename):
 
     def load_data(self, filename, data_path, indices_path, update_memory=True, is_mmap=True):
         with self.lock:
-            if not is_mmap:
-                with open(data_path / filename, 'rb') as f:
+            with open(data_path / filename, 'rb') as f:
                     data = np.load(f)
-                with open(indices_path / filename, 'rb') as f:
-                    indices = np.load(f)
-            else:
-                with NpyLoader(data_path / filename) as _data:
-                    data = np.asarray(_data)
-                with NpyLoader(indices_path / filename) as _indices:
-                    indices = np.asarray(_indices)
+            with open(indices_path / filename, 'rb') as f:
+                indices = np.load(f)
+
+            # ignore mmap
+            # if not is_mmap:
+            #     with open(data_path / filename, 'rb') as f:
+            #         data = np.load(f)
+            #     with open(indices_path / filename, 'rb') as f:
+            #         indices = np.load(f)
+            # else:
+            #     with NpyLoader(data_path / filename) as _data:
+            #         data = np.asarray(_data)
+            #     with NpyLoader(indices_path / filename) as _indices:
+            #         indices = np.asarray(_indices)
                     
         if update_memory:
             self.write_to_memory(filename, data, indices)
diff --git a/lynse/utils/utils.py b/lynse/utils/utils.py
index be7a1ad..8f53247 100644
--- a/lynse/utils/utils.py
+++ b/lynse/utils/utils.py
@@ -109,7 +109,7 @@ def wrapper(*args, **kwargs):
 
 
 def load_chunk_file(filename):
-    np_array = np.load(filename)
+    np_array = np.load(filename, mmap_mode='r')
     return np_array
 
 
@@ -117,21 +117,15 @@ class NpyLoader:
     def __init__(self, filename):
         self.filename = filename
         self.np_array = None
-        # self.mmap = None
 
     def __enter__(self):
         self.np_array = load_chunk_file(self.filename)
-
-        # self.mmap = getattr(self.np_array, 'base', None)
-
         return self.np_array
     
     def __exit__(self, exc_type, exc_value, traceback):
-        pass
-        # if self.np_array is not None:
-        #     del self.np_array
-        #     if self.mmap is not None and hasattr(self.mmap, 'close'):
-        #         self.mmap.close()
+        self.np_array._mmap.close()
+        self.np_array = None
+