diff --git a/src/constants.py b/src/constants.py index 4fb22a85..929c79e1 100644 --- a/src/constants.py +++ b/src/constants.py @@ -407,6 +407,16 @@ 'type': 'vector', 'precision': 'float16' }, + { + 'name': 'sentence-t5-xxl', + 'dimensions': 768, + 'max_sequence': 256, + 'size_mb': 9230, + 'repo_id': 'sentence-transformers/sentence-t5-xxl', + 'cache_dir': 'sentence-transformers--sentence-t5-xxl', + 'type': 'vector', + 'precision': 'float16' + }, ], 'thenlper': [ { @@ -929,7 +939,7 @@ "" "" "" -), + ), "VECTOR_MODEL_SELECT": "Choose a vector model to download.", "VECTOR_MODEL_SIZE": "Size on disk.", "VISION_MODEL": "Select vision model for image processing. Test before bulk processing.", @@ -938,8 +948,6 @@ "WHISPER_MODEL_SELECT": "Distil models use ~ 70% VRAM of their non-Distil equivalents with little quality loss." } - - scrape_documentation = { "Accelerate 0.34.2": { "URL": "https://huggingface.co/docs/accelerate/v0.34.2/en/", diff --git a/src/database_interactions.py b/src/database_interactions.py index a862e8ab..6bf20a0f 100644 --- a/src/database_interactions.py +++ b/src/database_interactions.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Optional import threading +import re import sqlite3 import torch @@ -23,7 +24,32 @@ def create_vector_db_in_process(database_name): create_vector_db = CreateVectorDB(database_name=database_name) create_vector_db.run() - + +def process_chunks_only_query(database_name, query, result_queue): + try: + query_db = QueryVectorDB(database_name) + contexts, metadata_list = query_db.search(query) + + formatted_contexts = [] + for index, (context, metadata) in enumerate(zip(contexts, metadata_list), start=1): + file_name = metadata.get('file_name', 'Unknown') + cleaned_context = re.sub(r'\n[ \t]+\n', '\n\n', context) + cleaned_context = re.sub(r'\n\s*\n\s*\n*', '\n\n', cleaned_context.strip()) + formatted_context = ( + f"{'-'*80}\n" + f"CONTEXT {index} | {file_name}\n" + f"{'-'*80}\n" + f"{cleaned_context}\n" + ) + formatted_contexts.append(formatted_context) + + result_queue.put("\n".join(formatted_contexts)) + except Exception as e: + result_queue.put(f"Error querying database: {str(e)}") + finally: + if 'query_db' in locals(): + query_db.cleanup() + class CreateVectorDB: def __init__(self, database_name): self.ROOT_DIRECTORY = Path(__file__).resolve().parent @@ -57,6 +83,7 @@ def initialize_vector_model(self, embedding_model_name, config_data): encode_kwargs['batch_size'] = 2 else: batch_size_mapping = { + 't5-xxl': 2, 't5-xl': 2, 't5-large': 4, 'instructor-xl': 2, @@ -354,7 +381,7 @@ def get_instance(cls, selected_database): with cls._instance_lock: if cls._instance is not None: if cls._instance.selected_database != selected_database: - logging.info(f"Database changed from {cls._instance.selected_database} to {selected_database}") + print(f"Database changed from {cls._instance.selected_database} to {selected_database}") cls._instance.cleanup() cls._instance = None else: @@ -366,7 +393,6 @@ def get_instance(cls, selected_database): return cls._instance def load_configuration(self): - """Load configuration from config.yaml file""" config_path = Path(__file__).resolve().parent / 'config.yaml' try: with open(config_path, 'r', encoding='utf-8') as file: @@ -439,7 +465,6 @@ def search(self, query, k: Optional[int] = None, score_threshold: Optional[float logging.info(f"Initializing database connection for {self.selected_database}") self.db = self.initialize_database() - # The rest of your existing search method remains unchanged self.config = self.load_configuration() document_types = self.config['database'].get('document_types', '') search_filter = {'document_type': document_types} if document_types else {} diff --git a/src/gui.py b/src/gui.py index 8f15822c..4a6d88ea 100644 --- a/src/gui.py +++ b/src/gui.py @@ -11,18 +11,16 @@ def set_cuda_paths(): cuda_path = nvidia_base_path / 'cuda_runtime' / 'bin' cublas_path = nvidia_base_path / 'cublas' / 'bin' cudnn_path = nvidia_base_path / 'cudnn' / 'bin' - # nvcc_path = nvidia_base_path / 'cuda_nvcc' / 'bin' nvrtc_path = nvidia_base_path / 'cuda_nvrtc' / 'bin' paths_to_add = [ str(cuda_path), # CUDA runtime str(cublas_path), # cuBLAS str(cudnn_path), # cuDNN - # str(nvcc_path), # NVIDIA CUDA compiler str(nvrtc_path), # NVIDIA runtime compiler ] - env_vars = ['CUDA_PATH', 'CUDA_PATH_V12_1', 'PATH'] + env_vars = ['CUDA_PATH', 'PATH'] for env_var in env_vars: current_value = os.environ.get(env_var, '') diff --git a/src/gui_tabs_database_query.py b/src/gui_tabs_database_query.py index 644ca157..bdeca8e0 100644 --- a/src/gui_tabs_database_query.py +++ b/src/gui_tabs_database_query.py @@ -15,41 +15,45 @@ from module_voice_recorder import VoiceRecorder from utilities import check_preconditions_for_submit_question, my_cprint from constants import TOOLTIPS -from database_interactions import QueryVectorDB +from database_interactions import QueryVectorDB, process_chunks_only_query current_dir = Path(__file__).resolve().parent input_text_file = str(current_dir / 'chat_history.txt') -class DatabaseQueryThread(QThread): +class ChunksOnlyThread(QThread): chunks_ready = Signal(str) def __init__(self, query, database_name): super().__init__() self.query = query self.database_name = database_name + self.process = None def run(self): try: - query_db = QueryVectorDB(self.database_name) - contexts, metadata_list = query_db.search(self.query) - formatted_chunks = self.format_chunks(contexts, metadata_list) - self.chunks_ready.emit(formatted_chunks) + result_queue = multiprocessing.Queue() + + self.process = multiprocessing.Process( + target=process_chunks_only_query, + args=(self.database_name, self.query, result_queue) + ) + self.process.start() + + result = result_queue.get() + self.chunks_ready.emit(result) + + self.process.join() + self.process = None + except Exception as e: - logging.exception(f"Error in database query thread: {e}") + logging.exception(f"Error in chunks only thread: {e}") self.chunks_ready.emit(f"Error querying database: {str(e)}") - @staticmethod - def format_chunks(contexts, metadata_list): - formatted_contexts = [] - for index, (context, metadata) in enumerate(zip(contexts, metadata_list), start=1): - file_name = metadata.get('file_name', 'Unknown') - formatted_context = ( - f"---------- Context {index} | From File: {file_name} ----------\n" - f"{context}\n" - ) - formatted_contexts.append(formatted_context) - return "\n".join(formatted_contexts) + def stop(self): + if self.process and self.process.is_alive(): + self.process.terminate() + self.process.join() def run_tts_in_process(config_path, input_text_file): from module_tts import run_tts # Import here to avoid potential circular imports @@ -250,9 +254,8 @@ def on_submit_button_clicked(self): chunks_only = self.chunks_only_checkbox.isChecked() selected_database = self.database_pulldown.currentText() - if chunks_only: # only get chunks - self.database_query_thread = DatabaseQueryThread(user_question, selected_database) + self.database_query_thread = ChunksOnlyThread(user_question, selected_database) self.database_query_thread.chunks_ready.connect(self.display_chunks) self.database_query_thread.finished.connect(self.on_database_query_finished) self.database_query_thread.start() diff --git a/src/gui_tabs_manage_databases.py b/src/gui_tabs_manage_databases.py index 7a33bab7..de0b8fe4 100644 --- a/src/gui_tabs_manage_databases.py +++ b/src/gui_tabs_manage_databases.py @@ -56,6 +56,7 @@ def __init__(self): self.database_info_layout = QHBoxLayout() self.database_info_label = QLabel("No database selected.") + self.database_info_label.setTextFormat(Qt.RichText) self.database_info_layout.addWidget(self.database_info_label) self.layout.addLayout(self.database_info_layout) @@ -122,7 +123,11 @@ def update_table_view_and_info_label(self, index): model_name = Path(model_path).name chunk_size = db_config.get('chunk_size', '') chunk_overlap = db_config.get('chunk_overlap', '') - info_text = f"DB name: \"{selected_database}\" | Created with \"{model_name}\" | Chunk size/overlap = {chunk_size} / {chunk_overlap}." + info_text = (f'Name: "{selected_database}" ' + f'| ' + f'Model: "{model_name}" ' + f'| ' + f'Chunk size/overlap: {chunk_size} / {chunk_overlap}') self.database_info_label.setText(info_text) else: self.database_info_label.setText("Configuration missing.") diff --git a/src/gui_tabs_tools_scrape.py b/src/gui_tabs_tools_scrape.py index 1083bfd3..577cc917 100644 --- a/src/gui_tabs_tools_scrape.py +++ b/src/gui_tabs_tools_scrape.py @@ -39,12 +39,13 @@ def init_ui(self): main_layout.addLayout(hbox) - self.status_label = QLabel("Pages scraped: 0") + self.status_label = QLabel() self.status_label.setTextFormat(Qt.RichText) self.status_label.setOpenExternalLinks(False) self.status_label.setToolTip("Click the links to open the folder containing scraped data.") self.status_label.linkActivated.connect(self.open_folder) self.status_label.setAlignment(Qt.AlignLeft | Qt.AlignVCenter) + self.status_label.setText('Pages scraped: 0') main_layout.addWidget(self.status_label) main_layout.addStretch() @@ -115,16 +116,20 @@ def start_scraping(self): self.thread.start() def update_status(self, status): - self.status_label.setText(f'Pages scraped: {status} Open Folder') + self.status_label.setText( + f'Pages scraped: {status} ' + f'Open Folder' + ) def scraping_finished(self): self.scrape_button.setEnabled(True) selected_doc = self.doc_combo.currentText() final_count = len([f for f in os.listdir(self.current_folder) if f.endswith('.html')]) - self.status_label.setText(f'Scraping {selected_doc} completed. Pages scraped: {final_count} Open Folder') - self.populate_combo_box() - if hasattr(self.worker, 'cleanup'): - self.worker.cleanup() + self.status_label.setText( + f'Scraping {selected_doc} completed. ' + f'Pages scraped: {final_count} ' + f'Open Folder' + ) def open_folder(self, link): if link == "open_folder": diff --git a/src/module_ask_jeeves.py b/src/module_ask_jeeves.py index 4748f8cb..82264fab 100644 --- a/src/module_ask_jeeves.py +++ b/src/module_ask_jeeves.py @@ -1,6 +1,5 @@ import json import subprocess -import sys import platform import signal import os @@ -12,8 +11,8 @@ import sseclient from huggingface_hub import snapshot_download from PySide6.QtWidgets import ( - QMainWindow, QWidget, QVBoxLayout, QHBoxLayout, - QTextEdit, QLineEdit, QCommandLinkButton, QMessageBox, + QMainWindow, QWidget, QVBoxLayout, + QTextEdit, QLineEdit, QMessageBox, QLabel, QApplication, QProgressDialog ) from PySide6.QtCore import QThread, Signal, QObject, Qt diff --git a/src/module_tts.py b/src/module_tts.py index 08e9f378..84c9d99a 100644 --- a/src/module_tts.py +++ b/src/module_tts.py @@ -104,10 +104,8 @@ def __init__(self): def initialize_model_and_processor(self): repository_id = "suno/bark" if self.config['size'] == 'normal' else f"suno/bark-{self.config['size']}" - # processor self.processor = AutoProcessor.from_pretrained(repository_id, cache_dir=CACHE_DIR) - # model self.model = BarkModel.from_pretrained( repository_id, torch_dtype=torch.float16, @@ -133,6 +131,9 @@ def process_text_to_audio(self, sentences): **inputs, use_cache=True, do_sample=True, + # temperature=0.2, + # top_k=50, + # top_p=0.95, pad_token_id=0, ) diff --git a/src/setup_windows.py b/src/setup_windows.py index 47ca1dd4..16b477d0 100644 --- a/src/setup_windows.py +++ b/src/setup_windows.py @@ -23,7 +23,6 @@ +--------------+------------------------------------------------------------------+ * Per the next table, "cu124" and "cu121" refer to compatibility with CUDA release 12.4.1 and 12.1.1 specifically. - ************************************************* * PyTorch Dependencies and Version Requirements ************************************************* @@ -45,7 +44,6 @@ * 12.4.127 and 12.4.5.8 stem from CUDA release 12.4.1 specifically * In other words, torch is not 100% compatible with CUDA 12.1.0 or 12.4.0, for example, or any other version. - ******************************************** * cuDNN and CUDA Compatibility Matrix ******************************************** @@ -53,7 +51,6 @@ # cuDNN is different... # First, according to Nvidia, cuDNN 8.9.2.26 is only compatible up to CUDA 12.2 # Second, for cuDNN 9+ Nvidia promises compatibility for all 12.x releases, but the cuDNN + CUDA combination still controls static linking: - +---------------+-----------------+-------------------------+ | cuDNN Version | Static Linking | No Static Linking | +---------------+-----------------+-------------------------+ @@ -69,7 +66,6 @@ +---------------+-----------------+-------------------------+ * 9.2+ continues the same trend - ********************************************* * PyTorch Official Release Compatibility ********************************************* @@ -88,7 +84,6 @@ | 2.2 | >=3.8, <=3.11, (3.12 exp.) | CUDA 11.8, CUDNN 8.7.0.84 | CUDA 12.1, CUDNN 8.9.2.26 | +-----------------+----------------------------+-------------------------------------------------+---------------------------+ - ********************************* * Xformers Version Compatibility ********************************* @@ -109,7 +104,8 @@ # old repo for windows triton: https://github.com/jakaline-dev/Triton_win/releases # new repo for windows triton: https://github.com/woct0rdho/triton-windows/releases - +# "Triton 3.1.0 works with torch >= 2.4.0, not 2.3.x." +# "The wheels are built against CUDA 12.5, and they should work with other CUDA 12.x." ***************************************** * Flash Attention 2 (FA2) Compatibility @@ -117,24 +113,29 @@ # this table represents the version of Torch and CUDA that FA2 is compatible with # based on flash-attention/.github/workflows/publish.yml -+-----------------------+---------------------------------------+----------------+ -| FA2 Version | Torch Versions Supported | CUDA Versions | -+--------------+-----------------------------------------------+----------------+ -| v2.7.1.post4 | 2.2.2, 2.3.1, 2.4.0, 2.5.1, 2.6.0.dev20241001 | 11.8.0, 12.3.2 | -| v2.7.1.post3 | 2.2.2, 2.3.1, 2.4.0, 2.5.1, 2.6.0.dev20241001 | 11.8.0, 12.3.2 | -| v2.7.1.post2 | 2.2.2, 2.3.1, 2.4.0, 2.5.1, 2.6.0.dev20241001 | 11.8.0, 12.3.2 | -| v2.7.1.post1 | 2.2.2, 2.3.1, 2.4.0, 2.5.1, 2.6.0.dev20241010 | 11.8.0, 12.4.1 | -| v2.7.1 | 2.2.2, 2.3.1, 2.4.0, 2.5.1, 2.6.0.dev20241010 | 11.8.0, 12.4.1 | -| v2.7.0.post2 | 2.2.2, 2.3.1, 2.4.0, 2.5.1 | 11.8.0, 12.4.1 | -| v2.7.0.post1 | 2.2.2, 2.3.1, 2.4.0, 2.5.1 | 11.8.0, 12.4.1 | -| v2.7.0 | 2.2.2, 2.3.1, 2.4.0, 2.5.1 | 11.8.0, 12.3.2 | -| v2.6.3 | 2.2.2, 2.3.1, 2.4.0 | 11.8.0, 12.3.2 | -| v2.6.2 | 2.2.2, 2.3.1, 2.4.0.dev20240527 | 11.8.0, 12.3.2 | -| v2.6.1 | 2.2.2, 2.3.1, 2.4.0.dev20240514 | 11.8.0, 12.3.2 | -| v2.6.0.post1 | 2.2.2, 2.3.1, 2.4.0.dev20240514 | 11.8.0, 12.2.2 | -| v2.6.0 | 2.2.2, 2.3.1, 2.4.0.dev20240512 | 11.8.0, 12.2.2 | -| v2.5.9.post1 | 2.2.2, 2.3.0, 2.4.0.dev20240407 | 11.8.0, 12.2.2 | -+--------------+-----------------------------------------------+----------------+ ++---------------+-----------------------------------------------+----------------+ +| FA2 Version | Torch Versions Supported | CUDA Versions | ++---------------+-----------------------------------------------+----------------+ +| v2.7.1.post4 | 2.2.2, 2.3.1, 2.4.0, 2.5.1, 2.6.0.dev20241001 | 11.8.0, 12.3.2 | +| v2.7.1.post3 | 2.2.2, 2.3.1, 2.4.0, 2.5.1, 2.6.0.dev20241001 | 11.8.0, 12.3.2 | +| v2.7.1.post2 | 2.2.2, 2.3.1, 2.4.0, 2.5.1, 2.6.0.dev20241001 | 11.8.0, 12.3.2 | +| v2.7.1.post1* | 2.2.2, 2.3.1, 2.4.0, 2.5.1, 2.6.0.dev20241010 | 11.8.0, 12.4.1 | +| v2.7.1 | 2.2.2, 2.3.1, 2.4.0, 2.5.1, 2.6.0.dev20241010 | 11.8.0, 12.4.1 | +| v2.7.0.post2* | 2.2.2, 2.3.1, 2.4.0, 2.5.1 | 11.8.0, 12.4.1 | +| v2.7.0.post1 | 2.2.2, 2.3.1, 2.4.0, 2.5.1 | 11.8.0, 12.4.1 | +| v2.7.0 | 2.2.2, 2.3.1, 2.4.0, 2.5.1 | 11.8.0, 12.3.2 | +| v2.6.3* | 2.2.2, 2.3.1, 2.4.0 | 11.8.0, 12.3.2 | +| v2.6.2 | 2.2.2, 2.3.1, 2.4.0.dev20240527 | 11.8.0, 12.3.2 | +| v2.6.1* | 2.2.2, 2.3.1, 2.4.0.dev20240514 | 11.8.0, 12.3.2 | +| v2.6.0.post1 | 2.2.2, 2.3.1, 2.4.0.dev20240514 | 11.8.0, 12.2.2 | +| v2.6.0 | 2.2.2, 2.3.1, 2.4.0.dev20240512 | 11.8.0, 12.2.2 | +| v2.5.9.post1* | 2.2.2, 2.3.0, 2.4.0.dev20240407 | 11.8.0, 12.2.2 | ++---------------+-----------------------------------------------+----------------+ +* has a corresponding Windows build +* https://github.com/bdashore3/flash-attention/releases/ + +Adding support for Python 3.12 will be a huge effort. +Suspect libraries sare langchain, pydantic, aiohttp, and possibly more. """ start_time = time.time() @@ -286,21 +287,23 @@ def install_libraries(libraries): ] other_libraries = [ - "accelerate==1.1.1", + "accelerate==1.2.0", "aiofiles==24.1.0", - "aiohappyeyeballs==2.4.3", - "aiohttp==3.10.10", + "aiohappyeyeballs==2.4.4", + "aiohttp==3.11.10", "aiosignal==1.3.1", "anndata==0.11.1", "annotated-types==0.7.0", "antlr4-python3-runtime==4.9.3", # omegaconf 2.3.0 requires 4.9.* - "anyio==4.6.2.post1", + "anyio==4.7.0", + "argcomplete==3.5.2", # required by minicpm3 chat, datamodel-code-generator 0.26.2 requires >=1.10,<4.0 "array_api_compat==1.9.1", "attrs==24.2.0", - "av==13.1.0", + "av==14.0.1", "backoff==2.2.1", "beautifulsoup4==4.12.3", - "bitsandbytes==0.44.1", + "bitsandbytes==0.45.0", + "black==24.10.0", # required by minicpm3 chat, datamodel-code-generator requires >=19.10b0 "braceexpand==0.1.7", "certifi==2024.8.30", "cffi==1.17.1", @@ -308,49 +311,56 @@ def install_libraries(libraries): "charset-normalizer==3.4.0", "chattts-fork==0.0.8", "click==8.1.7", - "cloudpickle==2.2.1", # tiledb-cloud 0.12.29 requires less than 3.0.0 + "cloudpickle==2.2.1", # tiledb-cloud 0.12.31 requires >=1.4.1,<3 "colorama==0.4.6", "coloredlogs==15.0.1", "ctranslate2==4.5.0", "cycler==0.12.1", "dataclasses-json==0.6.7", "datasets==3.1.0", - "datamodel-code-generator==0.26.3", + "datamodel-code-generator==0.26.2", # required by minicpm3 chat "deepdiff==8.0.1", - "dill==0.3.8", # datasets 3.1.0 requires less than 0.3.9 + "dill==0.3.8", # datasets 3.1.0 requires >=0.3.0,<0.3.9 "distro==1.9.0", + "dnspython==2.7.0", # required by minicpm3 chat "docx2txt==0.8", "einops==0.8.0", "einx==0.3.0", + "email-validator==2.2.0", # required by minicpm3 chat "emoji==2.14.0", "encodec==0.1.1", - "et-xmlfile==1.1.0", # openpyxl requires; hesitate to upgrade since openpyxl's most recent version pre-dates et-xmlfile 2+ - "fastcore==1.7.20", + "et-xmlfile==1.1.0", # openpyxl requires; caution...openpyxl 3.1.5 (6/28/2024) predates et-xmlfile 2.0.0 (10/25/2024) + "fastcore==1.7.24", "fastprogress==1.0.3", # only required by whisperspeech "filetype==1.2.0", "filelock==3.16.1", "frozendict==2.4.6", "frozenlist==1.5.0", - "fsspec==2024.5.0", # datasets 3.1.0 requires less than or equal to <=2024.9.0,>=2023.1.0 + "fsspec==2024.9.0", # datasets 3.1.0 requires >=2023.1.0,<=2024.9.0 + "genson==1.3.0", # required by minicpm3 chat; datamodel-code-generator requires >=1.2.1,<2.0 "greenlet==3.1.1", "gTTS==2.5.3", "h11==0.14.0", "h5py==3.12.1", "httpcore==1.0.7", - "httpx==0.27.2", + "httpx==0.28.1", "httpx-sse==0.4.0", - "huggingface-hub==0.26.2", + "huggingface-hub==0.26.5", # tokenizers 0.20.3 requires >=0.16.4,<1.0 "humanfriendly==10.0", "HyperPyYAML==1.2.2", "idna==3.10", "importlib_metadata==8.5.0", + "inflect==5.6.2", # required by minicpm3 chat; datamodel-code-generator 0.26.2 requires >=4.1.0,<6.0 + "isort==5.13.2", # required by minicpm3 chat; datamodel-code-generator requires >=4.3.21,<6.0 "InstructorEmbedding==1.0.1", - "Jinja2==3.1.4", # datamodel-code-generator 0.26.2 requires less than 4.0 - "jiter==0.7.1", # required by openai newer versions + "Jinja2==3.1.4", # datamodel-code-generator 0.26.2 requires >=2.10.1,<4.0 + "jiter==0.8.0", # required by openai newer versions "joblib==1.4.2", "jsonpatch==1.33", "jsonpath-python==1.0.6", "jsonpointer==3.0.0", + "jsonschema==4.23.0", # required by minicpm3 chat + "jsonschema-specifications==2024.10.1", # required by minicpm3 chat; jsonschema 4.23.0 requires >=2023.03.6 "kiwisolver==1.4.7", "langchain-community==0.2.17", "langchain-core==0.2.43", @@ -373,13 +383,13 @@ def install_libraries(libraries): "markdown-it-py==3.0.0", "MarkupSafe==3.0.2", "marshmallow==3.23.1", # dataclasses-json 0.6.7 requires less than 4.0 - "matplotlib==3.9.2", # uniquely requires pyparsing==3.1.2 cycler==0.12.1 kiwisolver==1.4.5 + "matplotlib==3.9.3", # uniquely requires pyparsing==3.1.2 cycler==0.12.1 kiwisolver==1.4.5 "mdurl==0.1.2", "more-itertools==10.5.0", "mpmath==1.3.0", # sympy 1.12.1 requires less than 1.4 "msg-parser==1.2.0", "multidict==6.1.0", - "multiprocess==0.70.16", # datasets 3.1.0 requires multiprocess less than 0.70.17 + "multiprocess==0.70.16", # datasets 3.1.0 requires <0.70.17 "mypy-extensions==1.0.0", "natsort==8.4.0", "nest-asyncio==1.6.0", @@ -408,24 +418,26 @@ def install_libraries(libraries): "nvidia-ml-py==12.560.30", "olefile==0.47", "omegaconf==2.3.0", - "openai==1.55.0", # only required by chat_lm_studio.py script + "openai==1.57.0", # only required by chat_lm_studio.py script "openai-whisper==20231117", # only required by whisper_s2t if using openai vanilla backend "openpyxl==3.1.5", "optimum==1.23.3", "ordered-set==4.1.0", "orderly-set==5.2.2", # deepdiff 8.0.1 requires 5.2.2 "orjson==3.10.12", - "packaging==24.1", + "packaging==24.2", "pandas==2.2.3", + "pathspec==0.12.1", # required by minicpm3 chat + "peft==0.14.0", # only required by mississippi model "pillow==11.0.0", "platformdirs==4.3.6", - "propcache==0.2.0", - "protobuf==5.28.3", + "propcache==0.2.1", + "protobuf==5.29.1", "psutil==6.1.0", - "pyarrow==18.0.0", + "pyarrow==18.1.0", "pyarrow-hotfix==0.6", "pycparser==2.22", - "pydantic==2.9.2", + "pydantic==2.9.2", # datamodel-code-generator requires >=1.10.0,<3.0,!=2.4.0 "pydantic_core==2.23.4", # pydantic 2.9.2 requires 2.23.4 # "pydantic-settings==2.6.1", # not sure if required... "Pygments==2.18.0", @@ -436,15 +448,17 @@ def install_libraries(libraries): "python-dateutil==2.9.0.post0", "python-docx==1.1.2", # "python-dotenv==1.0.1", # only required by pydantic-settings - "python-iso639==2024.4.27", + "python-iso639==2024.10.22", "python-magic==0.4.27", "pytz==2024.2", "PyYAML==6.0.2", "rapidfuzz==3.10.1", + "referencing==0.35.1", # required by minicpm3 chat "regex==2024.9.11", # 2024.11.6 is the newest version but pypi gives an error for some reason "requests==2.32.3", "requests-toolbelt==1.0.0", "rich==13.9.4", + "rpds-py==0.22.3", # required by minicpm3 chat "ruamel.yaml==0.18.6", "ruamel.yaml.clib==0.2.12", "safetensors==0.4.5", @@ -452,7 +466,7 @@ def install_libraries(libraries): "scipy==1.14.1", "sentence-transformers==3.0.1", "sentencepiece==0.2.0", - "six==1.16.0", + "six==1.17.0", "sniffio==1.3.1", "sounddevice==0.5.1", "soundfile==0.12.1", @@ -462,56 +476,41 @@ def install_libraries(libraries): "sseclient-py==1.8.0", "sympy==1.12.1", # anything above is not compatible with llava-next-vicuna vision models "tabulate==0.9.0", - "tblib==1.7.0", - "tenacity==8.5.0", # langchain requires less than 9.0.0 + "tblib==1.7.0", # tiledb-cloud requires >=1.7.0, <2.0.0 + "tenacity==8.5.0", # langchain requires >=8.1.0,<9.0.0,!=8.4.0 "termcolor==2.5.0", "threadpoolctl==3.5.0", "tiktoken==0.8.0", "tiledb==0.32.5", - "tiledb-cloud==0.12.29", + "tiledb-cloud==0.12.31", "tiledb-vector-search==0.10.3", - "timm==1.0.11", - "tokenizers==0.20.3", - "tqdm==4.66.6", - "transformers==4.46.3", + "timm==1.0.12", + "tokenizers==0.21.0", + "tqdm==4.67.1", + "transformers==4.47.0", "typing-inspect==0.9.0", "typing_extensions==4.12.2", "unstructured-client==0.24.1", "tzdata==2024.2", "urllib3==2.2.3", - "vector-quantize-pytorch==1.15.3", + "vector-quantize-pytorch==1.20.11", "vocos==0.1.0", "watchdog==6.0.0", - "webdataset==0.2.86", + # "webdataset==0.2.86", + "webdataset==0.2.100", "wrapt==1.17.0", "xformers==0.0.25.post1", # highly-specific to torch version - # "xformers==0.0.27.post2", # requires torch 2.4.0 + # "xformers==0.0.28.post1", # requires torch 2.4.1 "xlrd==2.0.1", "xxhash==3.5.0", # "yarl==1.12.0", # langchain-related libraries and optimum require less than 2.0 - "yarl==1.18.0", # langchain-related libraries and optimum require less than 2.0 + "yarl==1.18.3", # langchain-related libraries and optimum require less than 2.0 "zipp==3.21.0", - # the following are only required by minicpm3 chat model - "argcomplete==3.5.1", # datamodel-code-generator 0.26.2 requires >=1.10,<4.0 - "black==24.10.0", - "datamodel-code-generator==0.26.2", - "dnspython==2.7.0", - "email-validator==2.2.0", - "genson==1.3.0", # datamodel-code-generator requires less than 2.0 - "inflect==5.6.2", # datamodel-code-generator 0.26.2 requires >=4.1.0,<6.0 - "isort==5.13.2", # datamodel-code-generator requires less than 6.0 - "jsonschema==4.23.0", - "jsonschema-specifications==2024.10.1", # jsonschema 4.23.0 requires >=2023.03.6 - "pathspec==0.12.1", - "referencing==0.35.1", - "rpds-py==0.21.0", - # the following are only required by mississippi vision model - "peft==0.13.2", ] full_install_libraries = [ - "PySide6==6.8.0.2", - "pymupdf==1.24.14", + "PySide6==6.8.1", + "pymupdf==1.25.0", "unstructured==0.13.4" ]