From 5f31e250d43c05a6ab9cfafe16500a6b2fe07858 Mon Sep 17 00:00:00 2001 From: Isobel Daley Date: Thu, 4 Jul 2024 15:51:19 +0100 Subject: [PATCH 1/3] feat: make openAI model version configurable --- .env.django | 1 + .env.example | 1 + .env.test | 1 + core_api/src/dependencies.py | 28 +++++++++++++++++++-------- redbox-core/redbox/models/settings.py | 25 ++++++++++++++++-------- 5 files changed, 40 insertions(+), 16 deletions(-) diff --git a/.env.django b/.env.django index 8aa7ddaed..41bb42dd2 100644 --- a/.env.django +++ b/.env.django @@ -2,6 +2,7 @@ ANTHROPIC_API_KEY= OPENAI_API_KEY= +OPENAI_MODEL= LLM_MAX_TOKENS=1024 # === Development === diff --git a/.env.example b/.env.example index 9a78c01e5..ed2f1714a 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,7 @@ ANTHROPIC_API_KEY= OPENAI_API_KEY= +OPENAI_API_KEY= LLM_MAX_TOKENS=1024 # === Development === diff --git a/.env.test b/.env.test index 6fac0ca6e..0bc9191ce 100644 --- a/.env.test +++ b/.env.test @@ -2,6 +2,7 @@ ANTHROPIC_API_KEY= OPENAI_API_KEY= +OPENAI_MODEL= LLM_MAX_TOKENS=1024 # === AI === diff --git a/core_api/src/dependencies.py b/core_api/src/dependencies.py index db3eab95e..d90c8594e 100644 --- a/core_api/src/dependencies.py +++ b/core_api/src/dependencies.py @@ -12,13 +12,16 @@ from langchain_core.retrievers import BaseRetriever from langchain_core.runnables import ConfigurableField from langchain_elasticsearch import ApproxRetrievalStrategy, ElasticsearchStore - -from core_api.src.callbacks import LoggerCallbackHandler -from core_api.src.retriever import AllElasticsearchRetriever, ParameterisedElasticsearchRetriever from redbox.model_db import MODEL_PATH from redbox.models import Settings from redbox.storage import ElasticsearchStorageHandler +from core_api.src.callbacks import LoggerCallbackHandler +from core_api.src.retriever import ( + AllElasticsearchRetriever, + ParameterisedElasticsearchRetriever, +) + logging.basicConfig(level=logging.INFO) log = logging.getLogger() @@ -29,13 +32,17 @@ def get_env() -> Settings: @lru_cache(1) -def get_elasticsearch_client(env: Annotated[Settings, Depends(get_env)]) -> Elasticsearch: +def get_elasticsearch_client( + env: Annotated[Settings, Depends(get_env)] +) -> Elasticsearch: return env.elasticsearch_client() @lru_cache(1) def get_embedding_model(env: Annotated[Settings, Depends(get_env)]) -> Embeddings: - embedding_model = SentenceTransformerEmbeddings(model_name=env.embedding_model, cache_folder=MODEL_PATH) + embedding_model = SentenceTransformerEmbeddings( + model_name=env.embedding_model, cache_folder=MODEL_PATH + ) log.info("Loaded embedding model from environment: %s", env.embedding_model) return embedding_model @@ -72,7 +79,8 @@ def get_vector_store( @lru_cache(1) def get_parameterised_retriever( - env: Annotated[Settings, Depends(get_env)], es: Annotated[Elasticsearch, Depends(get_elasticsearch_client)] + env: Annotated[Settings, Depends(get_env)], + es: Annotated[Elasticsearch, Depends(get_elasticsearch_client)], ) -> BaseRetriever: """Creates an Elasticsearch retriever runnable. @@ -94,14 +102,17 @@ def get_parameterised_retriever( embedding_model=get_embedding_model(env), ).configurable_fields( params=ConfigurableField( - id="params", name="Retriever parameters", description="A dictionary of parameters to use for the retriever." + id="params", + name="Retriever parameters", + description="A dictionary of parameters to use for the retriever.", ) ) @lru_cache(1) def get_all_chunks_retriever( - env: Annotated[Settings, Depends(get_env)], es: Annotated[Elasticsearch, Depends(get_elasticsearch_client)] + env: Annotated[Settings, Depends(get_env)], + es: Annotated[Elasticsearch, Depends(get_elasticsearch_client)], ): return AllElasticsearchRetriever( es_client=es, @@ -120,6 +131,7 @@ def get_llm(env: Annotated[Settings, Depends(get_env)]) -> ChatLiteLLM: streaming=True, openai_key=env.openai_api_key, callbacks=[logger_callback], + model=env.openai_model, ) elif env.azure_openai_api_key is not None: log.info("Creating Azure LLM Client") diff --git a/redbox-core/redbox/models/settings.py b/redbox-core/redbox/models/settings.py index 2716eba79..fb1534980 100644 --- a/redbox-core/redbox/models/settings.py +++ b/redbox-core/redbox/models/settings.py @@ -10,9 +10,7 @@ log = logging.getLogger() -VANILLA_SYSTEM_PROMPT = ( - "You are an AI assistant called Redbox tasked with answering questions and providing information objectively." -) +VANILLA_SYSTEM_PROMPT = "You are an AI assistant called Redbox tasked with answering questions and providing information objectively." RETRIEVAL_SYSTEM_PROMPT = ( "Given the following conversation and extracted parts of a long document and a question, create a final answer. \n" @@ -66,15 +64,21 @@ VANILLA_QUESTION_PROMPT = "{question}\n=========\n Response: " -RETRIEVAL_QUESTION_PROMPT = "{question} \n=========\n{formatted_documents}\n=========\nFINAL ANSWER: " +RETRIEVAL_QUESTION_PROMPT = ( + "{question} \n=========\n{formatted_documents}\n=========\nFINAL ANSWER: " +) -SUMMARISATION_QUESTION_PROMPT = "Question: {question}. \n\n Documents: \n\n {documents} \n\n Answer: " +SUMMARISATION_QUESTION_PROMPT = ( + "Question: {question}. \n\n Documents: \n\n {documents} \n\n Answer: " +) MAP_QUESTION_PROMPT = "Question: {question}. " MAP_DOCUMENT_PROMPT = "\n\n Documents: \n\n {documents} \n\n Answer: " -REDUCE_QUESTION_PROMPT = "Question: {question}. \n\n Documents: \n\n {summaries} \n\n Answer: " +REDUCE_QUESTION_PROMPT = ( + "Question: {question}. \n\n Documents: \n\n {summaries} \n\n Answer: " +) CONDENSE_QUESTION_PROMPT = "{question}\n=========\n Standalone question: " @@ -136,6 +140,7 @@ class Settings(BaseSettings): anthropic_api_key: str | None = None openai_api_key: str | None = None + openai_model: str | None = None azure_openai_api_key: str | None = None azure_openai_endpoint: str | None = None @@ -183,7 +188,9 @@ class Settings(BaseSettings): dev_mode: bool = False superuser_email: str | None = None - model_config = SettingsConfigDict(env_file=".env", env_nested_delimiter="__", extra="allow", frozen=True) + model_config = SettingsConfigDict( + env_file=".env", env_nested_delimiter="__", extra="allow", frozen=True + ) def elasticsearch_client(self) -> Elasticsearch: if isinstance(self.elastic, ElasticLocalSettings): @@ -204,7 +211,9 @@ def elasticsearch_client(self) -> Elasticsearch: log.info("Cloud ID = %s", self.elastic.cloud_id) log.info("Elastic Cloud API Key = %s", self.elastic.api_key) - return Elasticsearch(cloud_id=self.elastic.cloud_id, api_key=self.elastic.api_key) + return Elasticsearch( + cloud_id=self.elastic.cloud_id, api_key=self.elastic.api_key + ) def s3_client(self): if self.object_store == "minio": From ef0993f78fcbf93cd3d41e1298f578ed24f22144 Mon Sep 17 00:00:00 2001 From: Isobel Daley Date: Thu, 4 Jul 2024 16:37:40 +0100 Subject: [PATCH 2/3] fix: remove auto-formatting changes --- core_api/src/dependencies.py | 27 ++++++++------------------- redbox-core/redbox/models/settings.py | 24 ++++++++---------------- 2 files changed, 16 insertions(+), 35 deletions(-) diff --git a/core_api/src/dependencies.py b/core_api/src/dependencies.py index d90c8594e..4c03d2837 100644 --- a/core_api/src/dependencies.py +++ b/core_api/src/dependencies.py @@ -12,16 +12,13 @@ from langchain_core.retrievers import BaseRetriever from langchain_core.runnables import ConfigurableField from langchain_elasticsearch import ApproxRetrievalStrategy, ElasticsearchStore + +from core_api.src.callbacks import LoggerCallbackHandler +from core_api.src.retriever import AllElasticsearchRetriever, ParameterisedElasticsearchRetriever from redbox.model_db import MODEL_PATH from redbox.models import Settings from redbox.storage import ElasticsearchStorageHandler -from core_api.src.callbacks import LoggerCallbackHandler -from core_api.src.retriever import ( - AllElasticsearchRetriever, - ParameterisedElasticsearchRetriever, -) - logging.basicConfig(level=logging.INFO) log = logging.getLogger() @@ -32,17 +29,13 @@ def get_env() -> Settings: @lru_cache(1) -def get_elasticsearch_client( - env: Annotated[Settings, Depends(get_env)] -) -> Elasticsearch: +def get_elasticsearch_client(env: Annotated[Settings, Depends(get_env)]) -> Elasticsearch: return env.elasticsearch_client() @lru_cache(1) def get_embedding_model(env: Annotated[Settings, Depends(get_env)]) -> Embeddings: - embedding_model = SentenceTransformerEmbeddings( - model_name=env.embedding_model, cache_folder=MODEL_PATH - ) + embedding_model = SentenceTransformerEmbeddings(model_name=env.embedding_model, cache_folder=MODEL_PATH) log.info("Loaded embedding model from environment: %s", env.embedding_model) return embedding_model @@ -79,8 +72,7 @@ def get_vector_store( @lru_cache(1) def get_parameterised_retriever( - env: Annotated[Settings, Depends(get_env)], - es: Annotated[Elasticsearch, Depends(get_elasticsearch_client)], + env: Annotated[Settings, Depends(get_env)], es: Annotated[Elasticsearch, Depends(get_elasticsearch_client)], ) -> BaseRetriever: """Creates an Elasticsearch retriever runnable. @@ -102,17 +94,14 @@ def get_parameterised_retriever( embedding_model=get_embedding_model(env), ).configurable_fields( params=ConfigurableField( - id="params", - name="Retriever parameters", - description="A dictionary of parameters to use for the retriever.", + id="params", name="Retriever parameters", description="A dictionary of parameters to use for the retriever.", ) ) @lru_cache(1) def get_all_chunks_retriever( - env: Annotated[Settings, Depends(get_env)], - es: Annotated[Elasticsearch, Depends(get_elasticsearch_client)], + env: Annotated[Settings, Depends(get_env)], es: Annotated[Elasticsearch, Depends(get_elasticsearch_client)], ): return AllElasticsearchRetriever( es_client=es, diff --git a/redbox-core/redbox/models/settings.py b/redbox-core/redbox/models/settings.py index fb1534980..f899fb77e 100644 --- a/redbox-core/redbox/models/settings.py +++ b/redbox-core/redbox/models/settings.py @@ -10,7 +10,9 @@ log = logging.getLogger() -VANILLA_SYSTEM_PROMPT = "You are an AI assistant called Redbox tasked with answering questions and providing information objectively." +VANILLA_SYSTEM_PROMPT = ( + "You are an AI assistant called Redbox tasked with answering questions and providing information objectively." +) RETRIEVAL_SYSTEM_PROMPT = ( "Given the following conversation and extracted parts of a long document and a question, create a final answer. \n" @@ -64,21 +66,15 @@ VANILLA_QUESTION_PROMPT = "{question}\n=========\n Response: " -RETRIEVAL_QUESTION_PROMPT = ( - "{question} \n=========\n{formatted_documents}\n=========\nFINAL ANSWER: " -) +RETRIEVAL_QUESTION_PROMPT = "{question} \n=========\n{formatted_documents}\n=========\nFINAL ANSWER: " -SUMMARISATION_QUESTION_PROMPT = ( - "Question: {question}. \n\n Documents: \n\n {documents} \n\n Answer: " -) +SUMMARISATION_QUESTION_PROMPT = "Question: {question}. \n\n Documents: \n\n {documents} \n\n Answer: " MAP_QUESTION_PROMPT = "Question: {question}. " MAP_DOCUMENT_PROMPT = "\n\n Documents: \n\n {documents} \n\n Answer: " -REDUCE_QUESTION_PROMPT = ( - "Question: {question}. \n\n Documents: \n\n {summaries} \n\n Answer: " -) +REDUCE_QUESTION_PROMPT = "Question: {question}. \n\n Documents: \n\n {summaries} \n\n Answer: " CONDENSE_QUESTION_PROMPT = "{question}\n=========\n Standalone question: " @@ -188,9 +184,7 @@ class Settings(BaseSettings): dev_mode: bool = False superuser_email: str | None = None - model_config = SettingsConfigDict( - env_file=".env", env_nested_delimiter="__", extra="allow", frozen=True - ) + model_config = SettingsConfigDict(env_file=".env", env_nested_delimiter="__", extra="allow", frozen=True) def elasticsearch_client(self) -> Elasticsearch: if isinstance(self.elastic, ElasticLocalSettings): @@ -211,9 +205,7 @@ def elasticsearch_client(self) -> Elasticsearch: log.info("Cloud ID = %s", self.elastic.cloud_id) log.info("Elastic Cloud API Key = %s", self.elastic.api_key) - return Elasticsearch( - cloud_id=self.elastic.cloud_id, api_key=self.elastic.api_key - ) + return Elasticsearch(cloud_id=self.elastic.cloud_id, api_key=self.elastic.api_key) def s3_client(self): if self.object_store == "minio": From 3d7ad66953c502946b79baa0f7e3c6b3039280f2 Mon Sep 17 00:00:00 2001 From: Isobel Daley Date: Thu, 4 Jul 2024 16:43:13 +0100 Subject: [PATCH 3/3] fix: remove auto-formatting changes --- core_api/src/dependencies.py | 6 +++--- redbox-core/redbox/models/settings.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core_api/src/dependencies.py b/core_api/src/dependencies.py index 4c03d2837..fd477fb77 100644 --- a/core_api/src/dependencies.py +++ b/core_api/src/dependencies.py @@ -72,7 +72,7 @@ def get_vector_store( @lru_cache(1) def get_parameterised_retriever( - env: Annotated[Settings, Depends(get_env)], es: Annotated[Elasticsearch, Depends(get_elasticsearch_client)], + env: Annotated[Settings, Depends(get_env)], es: Annotated[Elasticsearch, Depends(get_elasticsearch_client)] ) -> BaseRetriever: """Creates an Elasticsearch retriever runnable. @@ -94,14 +94,14 @@ def get_parameterised_retriever( embedding_model=get_embedding_model(env), ).configurable_fields( params=ConfigurableField( - id="params", name="Retriever parameters", description="A dictionary of parameters to use for the retriever.", + id="params", name="Retriever parameters", description="A dictionary of parameters to use for the retriever." ) ) @lru_cache(1) def get_all_chunks_retriever( - env: Annotated[Settings, Depends(get_env)], es: Annotated[Elasticsearch, Depends(get_elasticsearch_client)], + env: Annotated[Settings, Depends(get_env)], es: Annotated[Elasticsearch, Depends(get_elasticsearch_client)] ): return AllElasticsearchRetriever( es_client=es, diff --git a/redbox-core/redbox/models/settings.py b/redbox-core/redbox/models/settings.py index f899fb77e..b7f1d69f8 100644 --- a/redbox-core/redbox/models/settings.py +++ b/redbox-core/redbox/models/settings.py @@ -10,7 +10,7 @@ log = logging.getLogger() -VANILLA_SYSTEM_PROMPT = ( +VANILLA_SYSTEM_PROMPT = ( "You are an AI assistant called Redbox tasked with answering questions and providing information objectively." )