Small fix + more debug statements

danswer-ai · Sep 5, 2023 · 8c19c91 · 8c19c91
1 parent 15ea927
commit 8c19c91
Show file tree

Hide file tree

Showing 4 changed files with 15 additions and 7 deletions.
diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py
@@ -138,9 +138,9 @@
 #####
 NUM_RETURNED_HITS = 50
 NUM_RERANKED_RESULTS = 15
-# we feed in document chunks until we reach this token limit
-# default is ~5 full chunks (max chunk size is 512), although some chunks may be
-# smaller which could result in passing in more total chunks
+# We feed in document chunks until we reach this token limit.
+# Default is ~5 full chunks (max chunk size is 2000 chars), although some chunks
+# may be smaller which could result in passing in more total chunks
 NUM_DOCUMENT_TOKENS_FED_TO_GENERATIVE_MODEL = int(
     os.environ.get("NUM_DOCUMENT_TOKENS_FED_TO_GENERATIVE_MODEL") or (512 * 5)
 )

diff --git a/backend/danswer/direct_qa/answer_question.py b/backend/danswer/direct_qa/answer_question.py
@@ -114,6 +114,9 @@ def answer_qa_query(
         token_limit=NUM_DOCUMENT_TOKENS_FED_TO_GENERATIVE_MODEL,
         offset=offset_count,
     )
+    logger.debug(
+        f"Chunks fed to LLM: {[chunk.semantic_identifier for chunk in usable_chunks]}"
+    )
 
     error_msg = None
     try:

diff --git a/backend/danswer/direct_qa/qa_utils.py b/backend/danswer/direct_qa/qa_utils.py
@@ -268,6 +268,7 @@ def _get_usable_chunks(
         if total_token_count + chunk_token_count > token_limit:
             break
 
+        total_token_count += chunk_token_count
         usable_chunks.append(chunk)
 
     # try and return at least one chunk if possible. This chunk will
@@ -289,13 +290,14 @@ def get_usable_chunks(
     offset_into_chunks = 0
     usable_chunks: list[InferenceChunk] = []
     for _ in range(min(offset + 1, 1)):  # go through this process at least once
+        if offset_into_chunks >= len(usable_chunks) and offset_into_chunks > 0:
+            raise ValueError(
+                "Chunks offset too large, should not retry this many times"
+            )
+
         usable_chunks = _get_usable_chunks(
             chunks=chunks[offset_into_chunks:], token_limit=token_limit
         )
         offset_into_chunks += len(usable_chunks)
-        if offset_into_chunks >= len(usable_chunks):
-            raise ValueError(
-                "Chunks offset too large, should not retry this many times"
-            )
 
     return usable_chunks
diff --git a/backend/danswer/server/search_backend.py b/backend/danswer/server/search_backend.py
@@ -254,6 +254,9 @@ def stream_qa_portions(
             token_limit=NUM_DOCUMENT_TOKENS_FED_TO_GENERATIVE_MODEL,
             offset=offset_count,
         )
+        logger.debug(
+            f"Chunks fed to LLM: {[chunk.semantic_identifier for chunk in usable_chunks]}"
+        )
 
         try:
             for response_packet in qa_model.answer_question_stream(