From 9b52873b6a724800f44d688889b4b2343883cd9e Mon Sep 17 00:00:00 2001 From: langemittbacken Date: Mon, 18 Dec 2023 09:57:38 +0100 Subject: [PATCH 1/4] add fix for translate to en not working, init translation when pressing summarize and summarize no longer jumping --- backend/translation/app/translate.py | 2 +- frontend/src/TranscriptionDisplay.js | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/backend/translation/app/translate.py b/backend/translation/app/translate.py index 604f0ad..cebc176 100644 --- a/backend/translation/app/translate.py +++ b/backend/translation/app/translate.py @@ -28,7 +28,7 @@ def translate_to_lang(data: dict, from_language: str, to_language: str) -> dict: if from_language not in language_codes: translation = "Not a viable language" return translation - elif from_language != "en": + elif from_language != "en" and to_language != "en": install_language(from_language, "en") whole_text_translated = translate_json(data, from_language, "en") install_language("en", to_language) diff --git a/frontend/src/TranscriptionDisplay.js b/frontend/src/TranscriptionDisplay.js index f30b8cb..9d34ec9 100644 --- a/frontend/src/TranscriptionDisplay.js +++ b/frontend/src/TranscriptionDisplay.js @@ -71,7 +71,7 @@ function TranscriptionDisplay() { } let response = await fetch(endpoint); - if (!response.ok && selectedLanguage!==originalLanguage) { + if (!response.ok && selectedLanguage && selectedLanguage!==originalLanguage) { const userResponse = window.confirm("No translation found of , do you want to start a translation? (this might take some time)"); if (userResponse) { @@ -81,6 +81,7 @@ function TranscriptionDisplay() { } else { // User clicked "Cancel" console.log("User clicked Cancel"); + setSelectedLanguage("") return; } response = await fetch(endpoint, @@ -324,7 +325,7 @@ function TranscriptionDisplay() { Summarize ) : ( - data &&
{data.message}
+ data &&
{data.message}
) ) ) From 2ee84b1074f1015111d497286e367b57bb0c3968 Mon Sep 17 00:00:00 2001 From: tonytonfisk2 Date: Mon, 18 Dec 2023 16:13:46 +0100 Subject: [PATCH 2/4] small summary api fix --- backend/api/app/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/api/app/main.py b/backend/api/app/main.py index dfb266a..7117247 100644 --- a/backend/api/app/main.py +++ b/backend/api/app/main.py @@ -330,7 +330,7 @@ async def start_media_summary(media_id: str, background_tasks: BackgroundTasks): async def do_summary(file_path: str, media_id: str): - timeout_seconds = 300 + timeout_seconds = 600 session_timeout = aiohttp.ClientTimeout(total=timeout_seconds) summarize_url = f"http://{os.environ['SUMMARIZATION_ADDRESS']}:{os.environ['API_PORT_GUEST']}/summarize" summarize = {} @@ -347,7 +347,7 @@ async def do_summary(file_path: str, media_id: str): form_new.add_field('json_data', json.dumps(analysis_info), content_type='application/json') form_new.add_field('file', file) - async with aiohttp.request('POST', summarize_url, data=form_new) as response: + async with session.request('POST', summarize_url, data=form_new) as response: if response.status == status.HTTP_201_CREATED: summarize = await response.json() status_data = {"status": status.HTTP_200_OK, "message": "Summarization done."} From 281ec45da093de568eced0750ddb533f784f1f39 Mon Sep 17 00:00:00 2001 From: tonytonfisk2 Date: Mon, 18 Dec 2023 16:18:50 +0100 Subject: [PATCH 3/4] diarization change domain if more than 2ppl --- backend/diarization/app/diarize.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/backend/diarization/app/diarize.py b/backend/diarization/app/diarize.py index d16f202..a8685f0 100644 --- a/backend/diarization/app/diarize.py +++ b/backend/diarization/app/diarize.py @@ -39,7 +39,7 @@ def configurations(wav_path: str, domain: str, rttm: str | None, speakers: int = pretrained_msdd = "models/diar_msdd_telephonic.nemo" config.diarizer.manifest_filepath = input_manifest_path # config.device = device - config.batch_size = 1 + config.batch_size = 8 config.diarizer.out_dir = ut.OUTPUT_DIR # Directory to store intermediate files and prediction outputs config.diarizer.speaker_embeddings.model_path = pretrained_speaker_model config.diarizer.msdd_model.model_path = pretrained_msdd @@ -67,6 +67,15 @@ def msdd_diarization(config: OmegaConf): def create_diarization(file_path: str, rttm: str | None, speakers: int = None): - config = configurations(file_path, "telephonic", rttm, speakers) + domain = "" + if speakers is not None: + if speakers > 2: + domain = "meeting" + else: + domain = "telephonic" + else: + domain = "telephonic" + + config = configurations(file_path, domain, rttm, speakers) #cluster_diarization(config) msdd_diarization(config) \ No newline at end of file From f137ad4cfcb38c38b64758ce7c0087586a0b3cae Mon Sep 17 00:00:00 2001 From: tonytonfisk2 Date: Mon, 18 Dec 2023 16:22:29 +0100 Subject: [PATCH 4/4] summary, added a tokenizer or wont work. --- backend/diarization/app/diarize.py | 4 ++-- backend/summarization/app/summarize.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/diarization/app/diarize.py b/backend/diarization/app/diarize.py index a8685f0..4f23b98 100644 --- a/backend/diarization/app/diarize.py +++ b/backend/diarization/app/diarize.py @@ -70,9 +70,9 @@ def create_diarization(file_path: str, rttm: str | None, speakers: int = None): domain = "" if speakers is not None: if speakers > 2: - domain = "meeting" + domain = "meeting" #used for when there are 3+ ppl in the audio file else: - domain = "telephonic" + domain = "telephonic" #used for when its 2 ppl in the audio file else: domain = "telephonic" diff --git a/backend/summarization/app/summarize.py b/backend/summarization/app/summarize.py index adfd017..6d1cde9 100644 --- a/backend/summarization/app/summarize.py +++ b/backend/summarization/app/summarize.py @@ -1,10 +1,12 @@ -from llama_index import ListIndex, SimpleDirectoryReader, ServiceContext +from llama_index import ListIndex, SimpleDirectoryReader, ServiceContext, set_global_tokenizer from llama_index.llms import LlamaCPP from llama_index.llms.llama_utils import ( completion_to_prompt, ) import json, tempfile, os - +set_global_tokenizer( + AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1").encode +) def load_data(file_path: str): with open(file_path, 'r') as file: json_data = file.read()