diff --git a/lynse/execution_layer/indexer.py b/lynse/execution_layer/indexer.py index 737c00c..bbc8349 100644 --- a/lynse/execution_layer/indexer.py +++ b/lynse/execution_layer/indexer.py @@ -166,8 +166,7 @@ def build_binary(): f'{self.storage_worker.fingerprint}.*SQ8.index') ) - sq8_data = np.load(self.index_data_path / f'{self.storage_worker.fingerprint}.sqd', - mmap_mode='r') + sq8_data = np.load(self.index_data_path / f'{self.storage_worker.fingerprint}.sqd') _index.data = binary_data _index.ids = binary_ids diff --git a/lynse/storage_layer/storage.py b/lynse/storage_layer/storage.py index 1a154cd..9b7c462 100644 --- a/lynse/storage_layer/storage.py +++ b/lynse/storage_layer/storage.py @@ -262,16 +262,22 @@ def return_if_in_memory(self, filename): def load_data(self, filename, data_path, indices_path, update_memory=True, is_mmap=True): with self.lock: - if not is_mmap: - with open(data_path / filename, 'rb') as f: + with open(data_path / filename, 'rb') as f: data = np.load(f) - with open(indices_path / filename, 'rb') as f: - indices = np.load(f) - else: - with NpyLoader(data_path / filename) as _data: - data = np.asarray(_data) - with NpyLoader(indices_path / filename) as _indices: - indices = np.asarray(_indices) + with open(indices_path / filename, 'rb') as f: + indices = np.load(f) + + # ignore mmap + # if not is_mmap: + # with open(data_path / filename, 'rb') as f: + # data = np.load(f) + # with open(indices_path / filename, 'rb') as f: + # indices = np.load(f) + # else: + # with NpyLoader(data_path / filename) as _data: + # data = np.asarray(_data) + # with NpyLoader(indices_path / filename) as _indices: + # indices = np.asarray(_indices) if update_memory: self.write_to_memory(filename, data, indices) diff --git a/lynse/utils/utils.py b/lynse/utils/utils.py index be7a1ad..8f53247 100644 --- a/lynse/utils/utils.py +++ b/lynse/utils/utils.py @@ -109,7 +109,7 @@ def wrapper(*args, **kwargs): def load_chunk_file(filename): - np_array = np.load(filename) + np_array = np.load(filename, mmap_mode='r') return np_array @@ -117,21 +117,15 @@ class NpyLoader: def __init__(self, filename): self.filename = filename self.np_array = None - # self.mmap = None def __enter__(self): self.np_array = load_chunk_file(self.filename) - - # self.mmap = getattr(self.np_array, 'base', None) - return self.np_array def __exit__(self, exc_type, exc_value, traceback): - pass - # if self.np_array is not None: - # del self.np_array - # if self.mmap is not None and hasattr(self.mmap, 'close'): - # self.mmap.close() + self.np_array._mmap.close() + self.np_array = None +