From 2ef436f57e8b06548c12eefa5019d041888a1f7e Mon Sep 17 00:00:00 2001 From: Hugo Saporetti Junior Date: Wed, 20 Mar 2024 20:47:22 -0300 Subject: [PATCH] Added summarization QnA --- docs/devel/askai-questions.txt | 4 + src/main/askai/core/askai.py | 97 +++++++------------ src/main/askai/core/askai_messages.py | 34 ++++--- src/main/askai/core/component/audio_player.py | 2 +- src/main/askai/core/component/recorder.py | 12 ++- .../askai/core/processor/summary_processor.py | 53 +++++++--- .../askai/core/support/shared_instances.py | 35 ++++++- src/main/askai/core/support/utilities.py | 20 ++-- src/main/askai/language/argos_translator.py | 18 ++-- src/main/askai/language/teste.py | 20 ---- 10 files changed, 166 insertions(+), 129 deletions(-) create mode 100644 docs/devel/askai-questions.txt delete mode 100644 src/main/askai/language/teste.py diff --git a/docs/devel/askai-questions.txt b/docs/devel/askai-questions.txt new file mode 100644 index 00000000..45e53fd8 --- /dev/null +++ b/docs/devel/askai-questions.txt @@ -0,0 +1,4 @@ +Questions: + +1. summarize my markdown files at my HomeSetup docs folder. +2. \ No newline at end of file diff --git a/src/main/askai/core/askai.py b/src/main/askai/core/askai.py index 2c1ec441..c5f1eb32 100644 --- a/src/main/askai/core/askai.py +++ b/src/main/askai/core/askai.py @@ -20,23 +20,20 @@ import nltk import pause -from clitt.core.term.cursor import Cursor -from clitt.core.term.screen import Screen -from clitt.core.term.terminal import Terminal -from clitt.core.tui.line_input.line_input import line_input +from clitt.core.term.cursor import cursor +from clitt.core.term.screen import screen +from clitt.core.term.terminal import terminal from hspylib.core.enums.charset import Charset from hspylib.core.tools.commons import sysout from hspylib.modules.application.exit_status import ExitStatus -from hspylib.modules.cli.keyboard import Keyboard from hspylib.modules.eventbus.event import Event from askai.__classpath__ import classpath from askai.core.askai_configs import configs from askai.core.askai_events import ASKAI_BUS_NAME, AskAiEvents, REPLY_EVENT from askai.core.askai_messages import msg -from askai.core.askai_prompt import prompt -from askai.core.component.audio_player import AudioPlayer -from askai.core.component.cache_service import cache +from askai.core.component.audio_player import player +from askai.core.component.cache_service import cache, CACHE_DIR from askai.core.component.recorder import recorder from askai.core.engine.ai_engine import AIEngine from askai.core.model.chat_context import ChatContext @@ -99,14 +96,6 @@ def engine(self) -> AIEngine: def context(self) -> ChatContext: return self._context - @property - def nickname(self) -> str: - return f" {self.engine.nickname()}" - - @property - def username(self) -> str: - return f" {prompt.user.title()}" - @property def cache_enabled(self) -> bool: return configs.is_cache @@ -128,7 +117,7 @@ def is_processing(self, processing: bool) -> None: if processing: self.reply(msg.wait()) elif not processing and self._processing is not None and processing != self._processing: - Terminal.INSTANCE.cursor.erase_line() + terminal.cursor.erase_line() self._processing = processing def run(self) -> None: @@ -137,7 +126,7 @@ def run(self) -> None: self._startup() self._prompt() elif self.query_string: - display_text(f"{self.username}: {self.query_string}%EOL%") + display_text(f"{shared.username}: {self.query_string}%EOL%") self._ask_and_reply(self.query_string) def reply(self, message: str) -> None: @@ -145,21 +134,24 @@ def reply(self, message: str) -> None: :param message: The message to reply to the user. """ if self.is_speak: - self.engine.text_to_speech(f"{self.nickname}: ", message) + self.engine.text_to_speech(f"{shared.nickname}: ", message) else: - display_text(f"{self.nickname}: %GREEN%{message}%NC%") + display_text(f"{shared.nickname}: %GREEN%{message}%NC%") def reply_error(self, message: str) -> None: """Reply API or system errors. :param message: The error message to be displayed. """ log.error(message) - display_text(f"{self.nickname}: Error: {message or 'Aborted!'} %NC%") + if self.is_speak: + self.engine.text_to_speech(f"{shared.nickname}: ", message) + else: + display_text(f"{shared.nickname}: Error: {message or 'Aborted!'} %NC%") def _cb_reply_event(self, ev: Event) -> None: """Callback to handle reply events.""" if ev.args.erase_last: - Cursor.INSTANCE.erase_line() + cursor.erase_line() self.reply(ev.args.message) def _splash(self) -> None: @@ -167,11 +159,11 @@ def _splash(self) -> None: splash_interval = 1000 while not self._ready: if not self._processing: - Screen.INSTANCE.clear() + screen.clear() sysout(f"%GREEN%{self.SPLASH}%NC%") pause.milliseconds(splash_interval) pause.milliseconds(splash_interval * 2) - Screen.INSTANCE.clear() + screen.clear() def _startup(self) -> None: """Initialize the application.""" @@ -180,22 +172,22 @@ def _startup(self) -> None: if configs.is_speak: recorder.setup() configs.is_speak = recorder.input_device is not None - if configs.is_speak: - AudioPlayer.INSTANCE.start_delay() - nltk.download("averaged_perceptron_tagger", quiet=True) + nltk.download("averaged_perceptron_tagger", quiet=True, download_dir=CACHE_DIR) cache.set_cache_enable(self.cache_enabled) cache.read_query_history() askai_bus = AskAiEvents.get_bus(ASKAI_BUS_NAME) askai_bus.subscribe(REPLY_EVENT, self._cb_reply_event) + if configs.is_speak: + player.start_delay() self._ready = True - log.info("AskAI is ready !") splash_thread.join() display_text(self) + log.info("AskAI is ready to use!") self.reply(msg.welcome(os.getenv("USER", "you"))) def _prompt(self) -> None: """Prompt for user interaction.""" - while query := self._input(f"{self.username}: "): + while query := shared.input_text(f"{shared.username}: "): if not self._ask_and_reply(query): query = None break @@ -203,25 +195,6 @@ def _prompt(self) -> None: self.reply(msg.goodbye()) display_text("") - def _input(self, __prompt: str) -> Optional[str]: - """Prompt for user input. - :param __prompt: The prompt to display to the user. - """ - ret = None - while ret is None: - ret = line_input(__prompt) - if ret == Keyboard.VK_CTRL_L: # Use speech as input method. - Terminal.INSTANCE.cursor.erase_line() - spoken_text = self.engine.speech_to_text() - if spoken_text: - display_text(f"{self.username}: {spoken_text}") - ret = spoken_text - elif not isinstance(ret, str): - display_text(f"{self.username}: %YELLOW%Speech-To-Text is disabled!%NC%", erase_last=True) - ret = None - - return ret if not ret or isinstance(ret, str) else ret.val - def _ask_and_reply(self, question: str) -> bool: """Ask the question and provide the reply. :param question: The question to ask to the AI engine. @@ -229,19 +202,20 @@ def _ask_and_reply(self, question: str) -> bool: if not (reply := cache.read_reply(question)): log.debug('Response not found for "%s" in cache. Querying from %s.', question, self.engine.nickname()) status, response = proxy.process(question) - if status: - status = self._process_response(response) - else: - self.reply_error(response) + if status and response: + return self._process_response(response) + self.reply_error(response) else: - log.debug('Reply found for "%s" in cache.', question) + log.debug("Reply found for '%s' in cache.", question) self.reply(reply) status = True return status def _process_response(self, proxy_response: QueryResponse) -> bool: - """Process a query response using a processor that supports the query type.""" - status, output, q_type, processor = False, None, None, None + """Process a query response using a processor that supports the query type. + :param proxy_response: The processor proxy response. + """ + status, output, query_type, processor = False, None, None, None # Intrinsic features if not proxy_response.intelligible: self.reply_error(msg.intelligible(proxy_response.question)) @@ -258,23 +232,24 @@ def _process_response(self, proxy_response: QueryResponse) -> bool: processor = AIProcessor.get_by_name(SummaryProcessor.__name__) processor.bind(AIProcessor.get_by_name(GenericProcessor.__name__)) # Query processors - if processor or (q_type := proxy_response.query_type): - if not processor and not (processor := AIProcessor.get_by_query_type(q_type)): + if processor or (query_type := proxy_response.query_type): + if not processor and not (processor := AIProcessor.get_by_query_type(query_type)): log.error(f"Unable to find a proper processor: {str(proxy_response)}") - self.reply_error(msg.no_processor(q_type)) + self.reply_error(msg.no_processor(query_type)) return False log.info("%s::Processing response for '%s'", processor, proxy_response.question) status, output = processor.process(proxy_response) - if status and processor.next_in_chain(): + if status and output and processor.next_in_chain(): mapped_response = object_mapper.of_json(output, QueryResponse) if isinstance(mapped_response, QueryResponse): self._process_response(mapped_response) else: self.reply(str(mapped_response)) elif status: - self.reply(str(output)) + if output: + self.reply(output) else: - self.reply_error(str(output)) + self.reply_error(output) else: self.reply_error(msg.invalid_response(proxy_response)) diff --git a/src/main/askai/core/askai_messages.py b/src/main/askai/core/askai_messages.py index 7f0da3b6..7cca24cb 100644 --- a/src/main/askai/core/askai_messages.py +++ b/src/main/askai/core/askai_messages.py @@ -27,23 +27,23 @@ def welcome(self, username: str) -> str: @lru_cache def wait(self) -> str: - return self.translate(f"I'm thinking, please wait…") + return self.translate("I'm thinking, please wait…") @lru_cache - def listening(self) -> str: - return self.translate(f"I'm listening…") + def welcome_back(self) -> str: + return self.translate("Is there anything else I can help you with?") @lru_cache - def noise_levels(self) -> str: - return self.translate(f"Adjusting noise levels…") + def listening(self) -> str: + return self.translate("I'm listening…") @lru_cache def transcribing(self) -> str: - return self.translate(f"I'm processing your voice, please wait…") + return self.translate("I'm processing your voice, please wait…") @lru_cache def goodbye(self) -> str: - return self.translate(f"Goodbye, have a nice day ! ") + return self.translate("Goodbye, have a nice day ! ") @lru_cache def executing(self, cmd_line: str) -> str: @@ -63,25 +63,37 @@ def analysis_output(self) -> str: @lru_cache def searching(self) -> str: - return self.translate(f"Researching on Google…") + return self.translate("Researching on Google…") @lru_cache def summarizing(self, path: str) -> str: return self.translate(f"Summarizing documents from '{path}'. This can take a moment, please wait…") + @lru_cache + def enter_qna(self) -> str: + return self.translate("You entered the Summarization Questions and Answers") + + @lru_cache + def leave_qna(self) -> str: + return self.translate("You left the Summarization Questions and Answers") + + @lru_cache + def qna_welcome(self) -> str: + return self.translate("Question me about the summarized content") + # Warnings and alerts @lru_cache def cmd_no_output(self) -> str: - return self.translate(f"The command didn't return an output !") + return self.translate("The command didn't return an output !") @lru_cache def search_empty(self) -> str: - return self.translate(f"The google research didn't return an output !") + return self.translate("The google research didn't return an output !") @lru_cache def access_grant(self) -> str: - return self.translate(f"'AskAI' requires access to your files, folders and apps. Continue (yes/[no])?") + return self.translate("'AskAI' requires access to your files, folders and apps. Continue (yes/[no])?") @lru_cache def not_a_command(self, shell: str, content: str) -> str: diff --git a/src/main/askai/core/component/audio_player.py b/src/main/askai/core/component/audio_player.py index 253ee469..3f22f8b1 100644 --- a/src/main/askai/core/component/audio_player.py +++ b/src/main/askai/core/component/audio_player.py @@ -94,4 +94,4 @@ def play_sfx(self, filename: str, file_ext: Literal[".mp3", ".wav", ".m4a"] = ". return self.play_audio_file(filename) -assert AudioPlayer().INSTANCE is not None +assert (player := AudioPlayer().INSTANCE) is not None diff --git a/src/main/askai/core/component/recorder.py b/src/main/askai/core/component/recorder.py index 7e689ff6..2eab35bd 100644 --- a/src/main/askai/core/component/recorder.py +++ b/src/main/askai/core/component/recorder.py @@ -82,8 +82,10 @@ def input_device(self) -> Optional[Tuple[int, str]]: return self._input_device if self._input_device else None def listen( - self, recognition_api: RecognitionApi = RecognitionApi.OPEN_AI, language: Language = Language.EN_US - ) -> Tuple[Path, str]: + self, + recognition_api: RecognitionApi = RecognitionApi.OPEN_AI, + language: Language = Language.EN_US + ) -> Tuple[Path, Optional[str]]: """Listen to the microphone, save the AudioData as a wav file and then, transcribe the speech. :param recognition_api: the API to be used to recognize the speech. :param language: the spoken language. @@ -125,7 +127,7 @@ def _detect_noise(self, interval: float = 0.8) -> None: """ with Microphone() as source: try: - log.debug(msg.noise_levels()) + log.debug('Adjusting noise levels…') self._rec.adjust_for_ambient_noise(source, duration=interval) except UnknownValueError as err: raise IntelligibleAudioError(f"Unable to detect noise => {str(err)}") from err @@ -166,7 +168,9 @@ def _select_device(self) -> Optional[int]: return None def _test_device(self, idx: int) -> bool: - """TODO""" + """Test whether the input device specified by index can be used as an STT input. + :param idx: The index of the device to be tested. + """ log.debug(f"Testing input device at index: %d", idx) try: with Microphone(device_index=idx) as source: diff --git a/src/main/askai/core/processor/summary_processor.py b/src/main/askai/core/processor/summary_processor.py index b87046e2..7a437db3 100644 --- a/src/main/askai/core/processor/summary_processor.py +++ b/src/main/askai/core/processor/summary_processor.py @@ -12,9 +12,15 @@ Copyright·(c)·2024,·HSPyLib """ +import logging as log +import os +from typing import Optional, Tuple + +from langchain_core.prompts import PromptTemplate + +from askai.core.askai_events import AskAiEvents from askai.core.askai_messages import msg from askai.core.askai_prompt import prompt -from askai.core.component.cache_service import cache from askai.core.component.summarizer import summarizer from askai.core.engine.openai.temperatures import Temperatures from askai.core.model.chat_context import ContextRaw @@ -23,12 +29,8 @@ from askai.core.processor.ai_processor import AIProcessor from askai.core.support.object_mapper import object_mapper from askai.core.support.shared_instances import shared +from askai.core.support.utilities import display_text from askai.exception.exceptions import DocumentsNotFound -from langchain_core.prompts import PromptTemplate -from typing import Optional, Tuple - -import logging as log -import os class SummaryProcessor(AIProcessor): @@ -54,14 +56,12 @@ def process(self, query_response: QueryResponse) -> Tuple[bool, Optional[str]]: log.error(msg.invalid_response(SummaryResult)) output = response.message else: + shared.context.clear("SUMMARY") if not summarizer.exists(summary.folder, summary.glob): summarizer.generate(summary.folder, summary.glob) - if results := summarizer.query("Give me an overview of all the summarized content"): - output = os.linesep.join([r.answer for r in results]).strip() - shared.context.set("CONTEXT", output, "assistant") - cache.save_reply(query_response.question, output) - else: - log.info("Reusing existing summary: '%s'/'%s'", summary.folder, summary.glob) + else: + log.info("Reusing persisted summarized content: '%s'/'%s'", summary.folder, summary.glob) + output = self.qna() status = True else: output = msg.llm_error(response.message) @@ -70,3 +70,32 @@ def process(self, query_response: QueryResponse) -> Tuple[bool, Optional[str]]: status = True return status, output + + @staticmethod + def _ask_and_reply(question: str) -> Optional[str]: + """TODO""" + output = None + if results := summarizer.query(question): + output = os.linesep.join([r.answer for r in results]).strip() + shared.context.push("SUMMARY", question) + shared.context.push("SUMMARY", output, "assistant") + return output + + def qna(self) -> str: + """TODO""" + display_text( + f"%ORANGE%{'-=' * 40}%EOL%" + f"{msg.enter_qna()}%EOL%" + f"{'-=' * 40}%NC%" + ) + AskAiEvents.ASKAI_BUS.events.reply.emit(message=msg.qna_welcome()) + while question := shared.input_text(f"{shared.username}: %CYAN%"): + if not (output := self._ask_and_reply(question)): + break + AskAiEvents.ASKAI_BUS.events.reply.emit(message=f"%CYAN%{output}%NC%") + display_text( + f"%ORANGE%{'-=' * 40}%EOL%" + f"{msg.leave_qna()}%EOL%" + f"{'-=' * 40}%NC%" + ) + return msg.welcome_back() diff --git a/src/main/askai/core/support/shared_instances.py b/src/main/askai/core/support/shared_instances.py index 99ea5aba..88b267bb 100644 --- a/src/main/askai/core/support/shared_instances.py +++ b/src/main/askai/core/support/shared_instances.py @@ -1,9 +1,16 @@ +from typing import Optional + +from clitt.core.term.terminal import terminal +from clitt.core.tui.line_input.line_input import line_input +from hspylib.core.metaclass.singleton import Singleton +from hspylib.core.preconditions import check_state +from hspylib.modules.cli.keyboard import Keyboard + +from askai.core.askai_prompt import prompt from askai.core.engine.ai_engine import AIEngine from askai.core.engine.engine_factory import EngineFactory from askai.core.model.chat_context import ChatContext -from hspylib.core.metaclass.singleton import Singleton -from hspylib.core.preconditions import check_state -from typing import Optional +from askai.core.support.utilities import display_text class SharedInstances(metaclass=Singleton): @@ -33,6 +40,14 @@ def context(self, value: ChatContext) -> None: check_state(self._context is None, "Once set, this instance is immutable.") self._context = value + @property + def nickname(self) -> str: + return f" {self.engine.nickname()}" + + @property + def username(self) -> str: + return f" {prompt.user.title()}" + def create_engine(self, engine_name: str, model_name: str) -> AIEngine: """TODO""" if self._engine is None: @@ -45,5 +60,19 @@ def create_context(self, token_limit: int) -> ChatContext: self._context = ChatContext(token_limit) return self._context + def input_text(self, input_prompt: str) -> Optional[str]: + """Prompt for user input. + :param input_prompt: The prompt to display to the user. + """ + ret = None + while ret is None: + if (ret := line_input(input_prompt)) == Keyboard.VK_CTRL_L: # Use STT as input method. + terminal.cursor.erase_line() + if spoken_text := self.engine.speech_to_text(): + display_text(f"{self.username}: {spoken_text}") + ret = spoken_text + + return ret if not ret or isinstance(ret, str) else ret.val + assert (shared := SharedInstances().INSTANCE) is not None diff --git a/src/main/askai/core/support/utilities.py b/src/main/askai/core/support/utilities.py index 56bbc686..c0853aa7 100644 --- a/src/main/askai/core/support/utilities.py +++ b/src/main/askai/core/support/utilities.py @@ -12,22 +12,24 @@ Copyright·(c)·2024,·HSPyLib """ -from askai.core.support.presets import Presets -from askai.language.language import Language + +import hashlib +import os +import re +from os.path import basename, dirname +from pathlib import Path +from typing import Any, Optional, Tuple + +import pause from clitt.core.term.cursor import Cursor from hspylib.core.enums.charset import Charset from hspylib.core.preconditions import check_argument from hspylib.core.tools.commons import file_is_not_empty, sysout from hspylib.core.tools.text_tools import ensure_endswith from hspylib.modules.cli.vt100.vt_color import VtColor -from os.path import basename, dirname -from pathlib import Path -from typing import Any, Optional, Tuple -import hashlib -import os -import pause -import re +from askai.core.support.presets import Presets +from askai.language.language import Language CHAT_ICONS = { '': '\n\n%RED% Error: ', diff --git a/src/main/askai/language/argos_translator.py b/src/main/askai/language/argos_translator.py index 124c1ed5..4e550096 100644 --- a/src/main/askai/language/argos_translator.py +++ b/src/main/askai/language/argos_translator.py @@ -1,14 +1,16 @@ +import logging as log +import os +import sys +from functools import lru_cache +from typing import Optional + from argostranslate import package, translate from argostranslate.translate import ITranslation -from askai.exception.exceptions import TranslationPackageError -from askai.language.language import Language -from functools import lru_cache from hspylib.core.metaclass.singleton import Singleton -from typing import Optional -import logging as log -import os -import sys +from askai.core.component.cache_service import CACHE_DIR +from askai.exception.exceptions import TranslationPackageError +from askai.language.language import Language class ArgosTranslator(metaclass=Singleton): @@ -51,7 +53,7 @@ def __init__(self, from_idiom: Language, to_idiom: Language): argos_model = self._get_argos_model(from_idiom, to_idiom) self._argos_model = argos_model - @lru_cache(maxsize=500) + @lru_cache def translate(self, text: str) -> str: """Translate text using Argos translator. :param text: Text to translate. diff --git a/src/main/askai/language/teste.py b/src/main/askai/language/teste.py deleted file mode 100644 index 2536ce0e..00000000 --- a/src/main/askai/language/teste.py +++ /dev/null @@ -1,20 +0,0 @@ -import argostranslate.package -import argostranslate.translate - -from_code = "en" -to_code = "es" - -# Download and install Argos Translate package -argostranslate.package.update_package_index() -available_packages = argostranslate.package.get_available_packages() -package_to_install = next( - filter( - lambda x: x.from_code == from_code and x.to_code == to_code, available_packages - ) -) -argostranslate.package.install_from_path(package_to_install.download()) - -# Translate -translatedText = argostranslate.translate.translate("Hello World", from_code, to_code) -print(translatedText) -# '¡Hola Mundo!'