From 74d53205081f1a249a28f4bae3eb45cc873e84b8 Mon Sep 17 00:00:00 2001 From: Roy Miara Date: Wed, 4 Oct 2023 13:00:23 +0300 Subject: [PATCH 01/64] Add version in CLI: --- pyproject.toml | 1 + resin/__init__.py | 1 + resin_cli/cli.py | 13 ++++++++----- tests/unit/cli/test_basic.py | 16 ++++++++++++++++ 4 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 tests/unit/cli/test_basic.py diff --git a/pyproject.toml b/pyproject.toml index f350ffce..2ef12ea1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ pytest-mock = "^3.6.1" pytest-xdist = "^3.3.1" types-requests = "^2.31.0.2" httpx = "^0.25.0" +toml = "^0.10.2" [build-system] requires = ["poetry-core"] diff --git a/resin/__init__.py b/resin/__init__.py index e69de29b..541f859d 100644 --- a/resin/__init__.py +++ b/resin/__init__.py @@ -0,0 +1 @@ +__version__ = '0.1.0' \ No newline at end of file diff --git a/resin_cli/cli.py b/resin_cli/cli.py index 098cf1c8..da4f5de1 100644 --- a/resin_cli/cli.py +++ b/resin_cli/cli.py @@ -15,7 +15,9 @@ from resin_cli.data_loader import ( load_dataframe_from_path, IndexNotUniqueError, - DataframeValidationError) + DataframeValidationError +) +from resin import __version__ from .app import start as start_service from .cli_spinner import Spinner @@ -49,8 +51,9 @@ def validate_connection(): KnowledgeBase._connect_pinecone() except Exception: msg = ( - "Failed to connect to Pinecone index, please make sure" - + " you have set the right env vars" + "Error: Failed to connect to Pinecone index, please make sure" + + " you have set the right env vars" + + " PINECONE_API_KEY, INDEX_NAME, PINECONE_ENVIRONMENT" ) click.echo(click.style(msg, fg="red"), err=True) sys.exit(1) @@ -58,7 +61,7 @@ def validate_connection(): openai.Model.list() except Exception: msg = ( - "Failed to connect to OpenAI, please make sure" + "Error: Failed to connect to OpenAI, please make sure" + " you have set the right env vars" ) click.echo(click.style(msg, fg="red"), err=True) @@ -68,6 +71,7 @@ def validate_connection(): @click.group(invoke_without_command=True) +@click.version_option(__version__, '-v', '--version', prog_name='Resin') @click.pass_context def cli(ctx): """ @@ -78,7 +82,6 @@ def cli(ctx): if ctx.invoked_subcommand is None: validate_connection() click.echo(ctx.get_help()) - # click.echo(command.get_help(ctx)) @cli.command(help="Check if Resin service is running") diff --git a/tests/unit/cli/test_basic.py b/tests/unit/cli/test_basic.py new file mode 100644 index 00000000..b0d26fd1 --- /dev/null +++ b/tests/unit/cli/test_basic.py @@ -0,0 +1,16 @@ +import sys +from resin import __version__ + +def test_version(): + if sys.version_info > (3, 11): + import tomllib as toml + + with open("pyproject.toml", "rb") as f: + assert toml.load(f)["tool"]["poetry"]["version"] == __version__ + else: + import toml + + with open("pyproject.toml") as f: + assert toml.load(f)["tool"]["poetry"]["version"] == __version__ + + From 2c742a990d93709396d51c7cf61d63aef50f103c Mon Sep 17 00:00:00 2001 From: Roy Miara Date: Wed, 4 Oct 2023 16:17:54 +0300 Subject: [PATCH 02/64] minor change --- resin_cli/cli.py | 61 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 52 insertions(+), 9 deletions(-) diff --git a/resin_cli/cli.py b/resin_cli/cli.py index da4f5de1..73dbee14 100644 --- a/resin_cli/cli.py +++ b/resin_cli/cli.py @@ -112,9 +112,17 @@ def new(index_name, tokenizer_model): click.confirm(click.style("Do you want to continue?", fg="red"), abort=True) Tokenizer.initialize(OpenAITokenizer, tokenizer_model) with spinner: - _ = KnowledgeBase.create_with_new_index( - index_name=index_name - ) + try: + _ = KnowledgeBase.create_with_new_index( + index_name=index_name + ) + except Exception as e: #TODO: kb should throw a specific exception for each case + msg = ( + "Error: Failed to create a new index" + ) + click.echo(click.style(msg, fg="red"), err=True, nl=False) + click.echo(f" Reason: {e}") + sys.exit(1) click.echo(click.style("Success!", fg="green")) os.environ["INDEX_NAME"] = index_name @@ -133,7 +141,14 @@ def upsert(index_name, data_path, tokenizer_model): +' --index-name or set it with env var `export INDEX_NAME="MY_INDEX_NAME`' click.echo(click.style(msg, fg="red"), err=True) sys.exit(1) - Tokenizer.initialize(OpenAITokenizer, tokenizer_model) + try: + Tokenizer.initialize(OpenAITokenizer, tokenizer_model) + except Exception: + msg = ( + "Error: Failed to initialize tokenizer" + ) + click.echo(click.style(msg, fg="red"), err=True) + sys.exit(1) if data_path is None: msg = "Data path is not provided," + " please provide it with --data-path or set it with env var" @@ -144,7 +159,16 @@ def upsert(index_name, data_path, tokenizer_model): click.echo(" to index: ") click.echo(click.style(f'{INDEX_NAME_PREFIX}{index_name} \n', fg='green')) with spinner: - kb = KnowledgeBase(index_name=index_name) + try: + kb = KnowledgeBase(index_name=index_name) + except Exception: + msg = ( + "Error: Failed to connect to Pinecone index, please make sure" + + " you have set the right env vars" + + " PINECONE_API_KEY, INDEX_NAME, PINECONE_ENVIRONMENT" + ) + click.echo(click.style(msg, fg="red"), err=True) + sys.exit(1) try: data = load_dataframe_from_path(data_path) except IndexNotUniqueError: @@ -172,7 +196,17 @@ def upsert(index_name, data_path, tokenizer_model): pd.options.display.max_colwidth = 20 click.echo(data.head()) click.confirm(click.style("\nDoes this data look right?", fg="red"), abort=True) - kb.upsert_dataframe(data) + try: + kb.upsert_dataframe(data) + except Exception: + msg = ( + "Error: Failed to upsert data to index" + + f" {INDEX_NAME_PREFIX}{index_name}" + + " this could be due to connection issues" + + " please re-run `resin upsert`" + ) + click.echo(click.style(msg, fg="red"), err=True) + sys.exit(1) click.echo(click.style("Success!", fg="green")) @@ -189,9 +223,18 @@ def _chat( output = "" history += [{"role": "user", "content": message}] start = time.time() - openai_response = openai.ChatCompletion.create( - model=model, messages=history, stream=stream, api_base=api_base - ) + try: + openai_response = openai.ChatCompletion.create( + model=model, messages=history, stream=stream, api_base=api_base + ) + except Exception as e: + msg = ( + "Oops... something went wrong with the LLM" + + " the error I got is: " + ) + click.echo(click.style(msg, fg="red"), err=True, nl= False) + click.echo(f"{e}") + sys.exit(1) end = time.time() duration_in_sec = end - start click.echo(click.style(f"\n {speaker}:\n", fg=speaker_color)) From fce6376c8476eb9f38200cd739a290ce4888cbeb Mon Sep 17 00:00:00 2001 From: Roy Miara Date: Thu, 5 Oct 2023 09:53:15 +0300 Subject: [PATCH 03/64] Add misspelled test --- tests/unit/cli/test_data_loader.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/unit/cli/test_data_loader.py b/tests/unit/cli/test_data_loader.py index eeac43e5..9e58ecd1 100644 --- a/tests/unit/cli/test_data_loader.py +++ b/tests/unit/cli/test_data_loader.py @@ -104,6 +104,14 @@ ] ) +bad_df_missppelled_optional_field = pd.DataFrame( + [ + {"id": 1, "text": "foo", "sorce": "foo_source"}, + {"id": 2, "text": "bar", "metdata": {"key": "value"}}, + {"id": 3, "text": "baz", "sorce": "baz_source"}, + ] +) + bad_df_missing_mandatory_field = pd.DataFrame( [ {"text": "foo", "metadata": {"foo": "foo"}}, @@ -124,6 +132,7 @@ ("bad_df_bad_type_metadata_list_int", bad_df_bad_type_metadata_list_int), ("bad_df_has_excess_field", bad_df_has_excess_field), ("bad_df_missing_mandatory_field", bad_df_missing_mandatory_field), + ("bad_df_missppelled_optional_field", bad_df_missppelled_optional_field), ] From 8fde19f8a8e7c2174570767390d9403859587cb2 Mon Sep 17 00:00:00 2001 From: Roy Miara Date: Thu, 5 Oct 2023 09:59:48 +0300 Subject: [PATCH 04/64] add test cases --- resin/__init__.py | 2 +- resin_cli/cli.py | 2 +- tests/unit/cli/test_data_loader.py | 21 +++++++++++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/resin/__init__.py b/resin/__init__.py index 541f859d..b794fd40 100644 --- a/resin/__init__.py +++ b/resin/__init__.py @@ -1 +1 @@ -__version__ = '0.1.0' \ No newline at end of file +__version__ = '0.1.0' diff --git a/resin_cli/cli.py b/resin_cli/cli.py index 8743898e..3525bc3e 100644 --- a/resin_cli/cli.py +++ b/resin_cli/cli.py @@ -48,7 +48,7 @@ def validate_connection(): except Exception: msg = ( "Error: Failed to connect to Pinecone index, please make sure" - + " you have set the right env vars" + + " you have set the right env vars" + " PINECONE_API_KEY, INDEX_NAME, PINECONE_ENVIRONMENT" ) click.echo(click.style(msg, fg="red"), err=True) diff --git a/tests/unit/cli/test_data_loader.py b/tests/unit/cli/test_data_loader.py index 9e58ecd1..748a0103 100644 --- a/tests/unit/cli/test_data_loader.py +++ b/tests/unit/cli/test_data_loader.py @@ -22,6 +22,15 @@ ] ) +good_df_all_good_metadata_permutations = pd.DataFrame( + [ + {"id": 1, "text": "foo", "metadata": {"string": "string"}}, + {"id": 2, "text": "bar", "metadata": {"int": 1}}, + {"id": 3, "text": "baz", "metadata": {"float": 1.0}}, + {"id": 4, "text": "foo", "metadata": {"list": ["list", "another"]}}, + ] +) + good_df_maximal = pd.DataFrame( [ {"id": 1, "text": "foo", "source": "foo_source", "metadata": {"foo": "foo"}}, @@ -78,6 +87,17 @@ ] ) +bad_df_metadata_not_allowed_all_permutations = pd.DataFrame( + [ + {"id": 1, "text": "foo", "metadata": {"list_of_int": [1, 2, 3]}}, + {"id": 2, "text": "bar", "metadata": {"list_of_float": [1.0, 2.0, 3.0]}}, + {"id": 3, "text": "baz", "metadata": {"dict": {"key": "value"}}}, + {"id": 4, "text": "foo", "metadata": {"list_of_dict": [{"key": "value"}]}}, + {"id": 5, "text": "bar", "metadata": {"list_of_list": [["value"]]}}, + {"id": 6, "text": "baz", "metadata": {1: "foo"}}, + ] +) + bad_df_has_excess_field = pd.DataFrame( [ { @@ -133,6 +153,7 @@ ("bad_df_has_excess_field", bad_df_has_excess_field), ("bad_df_missing_mandatory_field", bad_df_missing_mandatory_field), ("bad_df_missppelled_optional_field", bad_df_missppelled_optional_field), + ("good_df_all_good_metadata_permutations", good_df_all_good_metadata_permutations), ] From 461a3eeafae000309c311137f2010082f1ffab10 Mon Sep 17 00:00:00 2001 From: Roy Miara Date: Thu, 5 Oct 2023 10:02:11 +0300 Subject: [PATCH 05/64] fix liniting --- resin_cli/cli.py | 5 +++-- tests/unit/cli/test_basic.py | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/resin_cli/cli.py b/resin_cli/cli.py index 3525bc3e..895c1ce1 100644 --- a/resin_cli/cli.py +++ b/resin_cli/cli.py @@ -112,7 +112,8 @@ def new(index_name, tokenizer_model): _ = KnowledgeBase.create_with_new_index( index_name=index_name ) - except Exception as e: #TODO: kb should throw a specific exception for each case + # TODO: kb should throw a specific exception for each case + except Exception as e: msg = ( "Error: Failed to create a new index" ) @@ -229,7 +230,7 @@ def _chat( "Oops... something went wrong with the LLM" + " the error I got is: " ) - click.echo(click.style(msg, fg="red"), err=True, nl= False) + click.echo(click.style(msg, fg="red"), err=True, nl=False) click.echo(f"{e}") sys.exit(1) end = time.time() diff --git a/tests/unit/cli/test_basic.py b/tests/unit/cli/test_basic.py index b0d26fd1..900df360 100644 --- a/tests/unit/cli/test_basic.py +++ b/tests/unit/cli/test_basic.py @@ -1,6 +1,7 @@ import sys from resin import __version__ + def test_version(): if sys.version_info > (3, 11): import tomllib as toml @@ -12,5 +13,3 @@ def test_version(): with open("pyproject.toml") as f: assert toml.load(f)["tool"]["poetry"]["version"] == __version__ - - From af7dc6af920406f233c0ac8f61f3fd08c3a5b512 Mon Sep 17 00:00:00 2001 From: Roy Miara Date: Thu, 5 Oct 2023 11:04:12 +0300 Subject: [PATCH 06/64] Added CLI help msgs --- resin_cli/cli.py | 75 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 68 insertions(+), 7 deletions(-) diff --git a/resin_cli/cli.py b/resin_cli/cli.py index 895c1ce1..b6270b81 100644 --- a/resin_cli/cli.py +++ b/resin_cli/cli.py @@ -80,7 +80,9 @@ def cli(ctx): click.echo(ctx.get_help()) -@cli.command(help="Check if Resin service is running") +@cli.command( + help="Check if service is running by sending a health check request" +) @click.option("--host", default="0.0.0.0", help="Host") @click.option("--port", default=8000, help="Port") @click.option("--ssl/--no-ssl", default=False, help="SSL") @@ -99,7 +101,14 @@ def health(host, port, ssl): return -@cli.command() +@cli.command( + help=( + "New command sets up a new index in Pinecone that is configured for Resin." + + " This will automatically tap the embedding model with a single toeken to " + + "assert for the dimensionality of the embedding space. This will also set up " + + "the index with the right schema for Resin." + ) +) @click.argument("index-name", nargs=1, envvar="INDEX_NAME", type=str, required=True) @click.option("--tokenizer-model", default="gpt-3.5-turbo", help="Tokenizer model") def new(index_name, tokenizer_model): @@ -124,7 +133,12 @@ def new(index_name, tokenizer_model): os.environ["INDEX_NAME"] = index_name -@cli.command() +@cli.command( + help=( + "Upsert allows you to load a loacal data file into a your Resin index." + + " The allowed formats are .jsonl and .parquet. The data will be validated" + ) +) @click.argument("data-path", type=click.Path(exists=True)) @click.option( "--index-name", @@ -270,7 +284,12 @@ def _chat( return debug_info -@cli.command() +@cli.command( + help=( + "Chat allows you to chat with your Resin index, " + + "Chat is a debugging tool, it is not meant to be used for production" + ) +) @click.option("--stream/--no-stream", default=True, help="Stream") @click.option("--debug/--no-debug", default=False, help="Print debug info") @click.option( @@ -292,6 +311,25 @@ def chat(index_name, chat_service_url, rag, debug, stream): ) click.echo(click.style(msg, fg="red"), err=True) sys.exit(1) + note_msg = ( + "🚨 Note 🚨\n" + + "Chat is a debugging tool, it is not meant to be used for production!" + ) + for c in note_msg: + click.echo(click.style(c, fg="red"), nl=False) + time.sleep(0.01) + click.echo() + note_white_message = ( + "This method should be used by developers to test the model and the data" + + " in development time, in local environment. " + + "For production use cases, we recommend using the" + + " Resin Service or Resin Library directly \n\n" + + "Let's Chat!" + ) + for c in note_white_message: + click.echo(click.style(c, fg="white"), nl=False) + time.sleep(0.01) + click.echo() history_with_pinecone = [] history_without_pinecone = [] @@ -333,17 +371,40 @@ def chat(index_name, chat_service_url, rag, debug, stream): click.echo(click.style("˙", fg="bright_black", bold=True)) -@cli.command() +@cli.command( + help=( + "Start the Resin service, this will start a uvicorn server" + + " that will serve the Resin API." + + " If you are are locally debugging, you can use the --debug flag" + + " to start a new terminal window with the right env vars for `resin chat`" + ) +) +@click.option("--debug/--no-debug", default=False, help="open a new terminal window for debugging") @click.option("--host", default="0.0.0.0", help="Host") @click.option("--port", default=8000, help="Port") @click.option("--ssl/--no-ssl", default=False, help="SSL") @click.option("--reload/--no-reload", default=False, help="Reload") -def start(host, port, ssl, reload): +def start(debug, host, port, ssl, reload): + if debug: + sys_msg = ( + f'open -na Terminal --env PINECONE_API_KEY="{os.environ["PINECONE_API_KEY"]}"' + + f' --env INDEX_NAME="{os.environ["INDEX_NAME"]}"' + + f' --env PINECONE_ENVIRONMENT="{os.environ["PINECONE_ENVIRONMENT"]}"' + + f' --env OPENAI_API_KEY="{os.environ["OPENAI_API_KEY"]}"' + ) + os.system(sys_msg) click.echo(f"Starting Resin service on {host}:{port}") start_service(host, port, reload) -@cli.command() +@cli.command( + help=( + "Stop the Resin service, this will kill the uvicorn server" + + " that is serving the Resin API." + + " This method is not recommended, as it will kill the server by looking for the PID" + + " of the server, instead, we recommend using ctrl+c on the terminal where you started" + ) +) @click.option("--host", default="0.0.0.0", help="Host") @click.option("--port", default=8000, help="Port") @click.option("--ssl/--no-ssl", default=False, help="SSL") From 036b29fd45a6e9fb263d56fae6e2a7de9909bb84 Mon Sep 17 00:00:00 2001 From: Roy Miara Date: Thu, 5 Oct 2023 11:21:11 +0300 Subject: [PATCH 07/64] edit chat msg --- resin_cli/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resin_cli/cli.py b/resin_cli/cli.py index b6270b81..efb20919 100644 --- a/resin_cli/cli.py +++ b/resin_cli/cli.py @@ -249,7 +249,7 @@ def _chat( sys.exit(1) end = time.time() duration_in_sec = end - start - click.echo(click.style(f"\n {speaker}:\n", fg=speaker_color)) + click.echo(click.style(f"\n> AI {speaker}:\n", fg=speaker_color)) if stream: for chunk in openai_response: openai_response_id = chunk.id From 4f3a57ecf7d7e24f2be8f34a6a9dbda0d4c58953 Mon Sep 17 00:00:00 2001 From: Roy Miara Date: Thu, 5 Oct 2023 11:24:22 +0300 Subject: [PATCH 08/64] fix linting --- resin_cli/cli.py | 44 +++++++++++++++++++------------------------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/resin_cli/cli.py b/resin_cli/cli.py index efb20919..b4452c44 100644 --- a/resin_cli/cli.py +++ b/resin_cli/cli.py @@ -16,7 +16,7 @@ from resin_cli.data_loader import ( load_dataframe_from_path, IndexNotUniqueError, - DataframeValidationError + DataframeValidationError, ) from resin import __version__ @@ -67,7 +67,7 @@ def validate_connection(): @click.group(invoke_without_command=True) -@click.version_option(__version__, '-v', '--version', prog_name='Resin') +@click.version_option(__version__, "-v", "--version", prog_name="Resin") @click.pass_context def cli(ctx): """ @@ -80,9 +80,7 @@ def cli(ctx): click.echo(ctx.get_help()) -@cli.command( - help="Check if service is running by sending a health check request" -) +@cli.command(help="Check if service is running by sending a health check request") @click.option("--host", default="0.0.0.0", help="Host") @click.option("--port", default=8000, help="Port") @click.option("--ssl/--no-ssl", default=False, help="SSL") @@ -118,14 +116,10 @@ def new(index_name, tokenizer_model): Tokenizer.initialize(OpenAITokenizer, tokenizer_model) with spinner: try: - _ = KnowledgeBase.create_with_new_index( - index_name=index_name - ) + _ = KnowledgeBase.create_with_new_index(index_name=index_name) # TODO: kb should throw a specific exception for each case except Exception as e: - msg = ( - "Error: Failed to create a new index" - ) + msg = "Error: Failed to create a new index" click.echo(click.style(msg, fg="red"), err=True, nl=False) click.echo(f" Reason: {e}") sys.exit(1) @@ -155,20 +149,18 @@ def upsert(index_name, data_path, tokenizer_model): try: Tokenizer.initialize(OpenAITokenizer, tokenizer_model) except Exception: - msg = ( - "Error: Failed to initialize tokenizer" - ) + msg = "Error: Failed to initialize tokenizer" click.echo(click.style(msg, fg="red"), err=True) sys.exit(1) if data_path is None: msg = "Data path is not provided," - + " please provide it with --data-path or set it with env var" + +" please provide it with --data-path or set it with env var" click.echo(click.style(msg, fg="red"), err=True) sys.exit(1) click.echo("Resin is going to upsert data from ", nl=False) - click.echo(click.style(f'{data_path}', fg='yellow'), nl=False) + click.echo(click.style(f"{data_path}", fg="yellow"), nl=False) click.echo(" to index: ") - click.echo(click.style(f'{INDEX_NAME_PREFIX}{index_name} \n', fg='green')) + click.echo(click.style(f"{INDEX_NAME_PREFIX}{index_name} \n", fg="green")) with spinner: try: kb = KnowledgeBase(index_name=index_name) @@ -240,10 +232,7 @@ def _chat( model=model, messages=history, stream=stream, api_base=api_base ) except Exception as e: - msg = ( - "Oops... something went wrong with the LLM" - + " the error I got is: " - ) + msg = "Oops... something went wrong with the LLM" + " the error I got is: " click.echo(click.style(msg, fg="red"), err=True, nl=False) click.echo(f"{e}") sys.exit(1) @@ -379,7 +368,9 @@ def chat(index_name, chat_service_url, rag, debug, stream): + " to start a new terminal window with the right env vars for `resin chat`" ) ) -@click.option("--debug/--no-debug", default=False, help="open a new terminal window for debugging") +@click.option( + "--debug/--no-debug", default=False, help="open a new terminal window for debugging" +) @click.option("--host", default="0.0.0.0", help="Host") @click.option("--port", default=8000, help="Port") @click.option("--ssl/--no-ssl", default=False, help="SSL") @@ -387,7 +378,8 @@ def chat(index_name, chat_service_url, rag, debug, stream): def start(debug, host, port, ssl, reload): if debug: sys_msg = ( - f'open -na Terminal --env PINECONE_API_KEY="{os.environ["PINECONE_API_KEY"]}"' + 'open -na Terminal' + + f' --env PINECONE_API_KEY="{os.environ["PINECONE_API_KEY"]}"' + f' --env INDEX_NAME="{os.environ["INDEX_NAME"]}"' + f' --env PINECONE_ENVIRONMENT="{os.environ["PINECONE_ENVIRONMENT"]}"' + f' --env OPENAI_API_KEY="{os.environ["OPENAI_API_KEY"]}"' @@ -401,8 +393,10 @@ def start(debug, host, port, ssl, reload): help=( "Stop the Resin service, this will kill the uvicorn server" + " that is serving the Resin API." - + " This method is not recommended, as it will kill the server by looking for the PID" - + " of the server, instead, we recommend using ctrl+c on the terminal where you started" + + " This method is not recommended," + + " as it will kill the server by looking for the PID" + + " of the server, instead, we recommend using" + + " ctrl+c on the terminal where you started" ) ) @click.option("--host", default="0.0.0.0", help="Host") From ca2df8ddad12d1436e44d48a034c5b0e9b35eed0 Mon Sep 17 00:00:00 2001 From: Roy Miara Date: Sun, 8 Oct 2023 16:53:37 +0300 Subject: [PATCH 09/64] fix resin chat --- resin_cli/cli.py | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/resin_cli/cli.py b/resin_cli/cli.py index b4452c44..cd2e6119 100644 --- a/resin_cli/cli.py +++ b/resin_cli/cli.py @@ -2,9 +2,11 @@ import click import time import sys +import subprocess import requests from dotenv import load_dotenv +from tenacity import retry, stop_after_attempt, wait_fixed import pandas as pd import openai @@ -41,6 +43,15 @@ def is_healthy(url: str): except Exception: return False +@retry(wait=wait_fixed(5), stop=stop_after_attempt(6)) +def wait_for_service(url: str): + if not is_healthy(chat_service_url): + msg = ( + f"Resin service is not running! on {chat_service_url}" + + " please run `resin start`" + ) + click.echo(click.style(msg, fg="red"), err=True) + sys.exit(1) def validate_connection(): try: @@ -369,22 +380,25 @@ def chat(index_name, chat_service_url, rag, debug, stream): ) ) @click.option( - "--debug/--no-debug", default=False, help="open a new terminal window for debugging" + "--chat", is_flag=True ,help="open a new terminal window for debugging" ) @click.option("--host", default="0.0.0.0", help="Host") @click.option("--port", default=8000, help="Port") @click.option("--ssl/--no-ssl", default=False, help="SSL") @click.option("--reload/--no-reload", default=False, help="Reload") -def start(debug, host, port, ssl, reload): - if debug: - sys_msg = ( - 'open -na Terminal' - + f' --env PINECONE_API_KEY="{os.environ["PINECONE_API_KEY"]}"' - + f' --env INDEX_NAME="{os.environ["INDEX_NAME"]}"' - + f' --env PINECONE_ENVIRONMENT="{os.environ["PINECONE_ENVIRONMENT"]}"' - + f' --env OPENAI_API_KEY="{os.environ["OPENAI_API_KEY"]}"' - ) - os.system(sys_msg) +def start(chat, host, port, ssl, reload): + if chat: + command_to_run = f"clear && echo Welcome to Pinecone Canopy," + + " run *resin chat* to start chatting with your index" + + script = f''' + tell application "Terminal" + activate + do script "{command_to_run}" + end tell + ''' + + subprocess.run(["osascript", "-e", script], env=os.environ.copy()) click.echo(f"Starting Resin service on {host}:{port}") start_service(host, port, reload) From c7a8d2f1a94f8e40931a1edd4ea7b62a9d33449e Mon Sep 17 00:00:00 2001 From: Roy Miara Date: Sun, 8 Oct 2023 17:28:37 +0300 Subject: [PATCH 10/64] lint --- resin_cli/cli.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/resin_cli/cli.py b/resin_cli/cli.py index cd2e6119..c889b3da 100644 --- a/resin_cli/cli.py +++ b/resin_cli/cli.py @@ -43,8 +43,9 @@ def is_healthy(url: str): except Exception: return False + @retry(wait=wait_fixed(5), stop=stop_after_attempt(6)) -def wait_for_service(url: str): +def wait_for_service(chat_service_url: str): if not is_healthy(chat_service_url): msg = ( f"Resin service is not running! on {chat_service_url}" @@ -53,6 +54,7 @@ def wait_for_service(url: str): click.echo(click.style(msg, fg="red"), err=True) sys.exit(1) + def validate_connection(): try: KnowledgeBase._connect_pinecone() @@ -380,7 +382,7 @@ def chat(index_name, chat_service_url, rag, debug, stream): ) ) @click.option( - "--chat", is_flag=True ,help="open a new terminal window for debugging" + "--chat", is_flag=True, help="open a new terminal window for debugging" ) @click.option("--host", default="0.0.0.0", help="Host") @click.option("--port", default=8000, help="Port") @@ -388,7 +390,7 @@ def chat(index_name, chat_service_url, rag, debug, stream): @click.option("--reload/--no-reload", default=False, help="Reload") def start(chat, host, port, ssl, reload): if chat: - command_to_run = f"clear && echo Welcome to Pinecone Canopy," + command_to_run = "clear && echo Welcome to Pinecone Canopy," + " run *resin chat* to start chatting with your index" script = f''' From 649e3f50cac5ed90d59215306b994129a496dcbb Mon Sep 17 00:00:00 2001 From: ilai Date: Mon, 16 Oct 2023 17:05:21 +0300 Subject: [PATCH 11/64] Proper solution for publishing package __version__ Since the package is installed, we can read the right version using `importlib --- pyproject.toml | 1 - resin/__init__.py | 5 ++++- tests/unit/cli/test_basic.py | 15 --------------- 3 files changed, 4 insertions(+), 17 deletions(-) delete mode 100644 tests/unit/cli/test_basic.py diff --git a/pyproject.toml b/pyproject.toml index 2ef12ea1..f350ffce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,6 @@ pytest-mock = "^3.6.1" pytest-xdist = "^3.3.1" types-requests = "^2.31.0.2" httpx = "^0.25.0" -toml = "^0.10.2" [build-system] requires = ["poetry-core"] diff --git a/resin/__init__.py b/resin/__init__.py index b794fd40..4a604304 100644 --- a/resin/__init__.py +++ b/resin/__init__.py @@ -1 +1,4 @@ -__version__ = '0.1.0' +import importlib.metadata + +# Taken from https://stackoverflow.com/a/67097076 +__version__ = importlib.metadata.version("pinecone-resin") diff --git a/tests/unit/cli/test_basic.py b/tests/unit/cli/test_basic.py deleted file mode 100644 index 900df360..00000000 --- a/tests/unit/cli/test_basic.py +++ /dev/null @@ -1,15 +0,0 @@ -import sys -from resin import __version__ - - -def test_version(): - if sys.version_info > (3, 11): - import tomllib as toml - - with open("pyproject.toml", "rb") as f: - assert toml.load(f)["tool"]["poetry"]["version"] == __version__ - else: - import toml - - with open("pyproject.toml") as f: - assert toml.load(f)["tool"]["poetry"]["version"] == __version__ From 26907246077b6b7f0ed0773ee37fb26919b263ea Mon Sep 17 00:00:00 2001 From: ilai Date: Mon, 16 Oct 2023 18:56:12 +0300 Subject: [PATCH 12/64] [cli] Simplify health check A. Changed is_healthy to simply exit, instead of being a bool - saves a lot of code duplication B. Distinguish between the case of service not running to service is running but unhealthy --- resin/__init__.py | 4 --- src/resin_cli/cli.py | 66 ++++++++++++++++++-------------------------- 2 files changed, 27 insertions(+), 43 deletions(-) delete mode 100644 resin/__init__.py diff --git a/resin/__init__.py b/resin/__init__.py deleted file mode 100644 index 4a604304..00000000 --- a/resin/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -import importlib.metadata - -# Taken from https://stackoverflow.com/a/67097076 -__version__ = importlib.metadata.version("pinecone-resin") diff --git a/src/resin_cli/cli.py b/src/resin_cli/cli.py index a0841efb..4d35aa50 100644 --- a/src/resin_cli/cli.py +++ b/src/resin_cli/cli.py @@ -1,4 +1,5 @@ import os +from textwrap import dedent import click import time @@ -30,31 +31,36 @@ dotenv_path = os.path.join(os.path.dirname(__file__), ".env") load_dotenv(dotenv_path) - spinner = Spinner() +format_multiline = lambda s: dedent(s).strip() -def is_healthy(url: str): +def check_service_health(url: str): try: health_url = os.path.join(url, "health") res = requests.get(health_url) res.raise_for_status() return res.ok - except Exception: - return False - - -@retry(wait=wait_fixed(5), stop=stop_after_attempt(6)) -def wait_for_service(chat_service_url: str): - if not is_healthy(chat_service_url): + except requests.exceptions.ConnectionError: + msg = f""" + Resin service is not running on {url}. + please run `resin start`" + """ + click.echo(click.style(format_multiline(msg), fg="red"), err=True) + sys.exit(1) + except Exception as e: msg = ( - f"Resin service is not running! on {chat_service_url}" - + " please run `resin start`" + f"Resin service on {url} is not healthy, failed with error: {e}" ) click.echo(click.style(msg, fg="red"), err=True) sys.exit(1) +@retry(wait=wait_fixed(5), stop=stop_after_attempt(6)) +def wait_for_service(chat_service_url: str): + check_service_health(chat_service_url) + + def validate_connection(): try: KnowledgeBase._connect_pinecone() @@ -93,23 +99,17 @@ def cli(ctx): click.echo(ctx.get_help()) -@cli.command(help="Check if service is running by sending a health check request") -@click.option("--host", default="0.0.0.0", help="Host") -@click.option("--port", default=8000, help="Port") -@click.option("--ssl/--no-ssl", default=False, help="SSL") +@cli.command(help="Check if resin service is running by sending a health check request") +@click.option("--host", default="0.0.0.0", help="Resin's service hostname") +@click.option("--port", default=8000, help="The port of the resin service") +@click.option("--ssl/--no-ssl", default=False, help="Whether to use ssl for the " + "connection to resin service") def health(host, port, ssl): ssl_str = "s" if ssl else "" service_url = f"http{ssl_str}://{host}:{port}" - if not is_healthy(service_url): - msg = ( - f"Resin service is not running! on {service_url}" - + " please run `resin start`" - ) - click.echo(click.style(msg, fg="red"), err=True) - sys.exit(1) - else: - click.echo(click.style("Resin service is healthy!", fg="green")) - return + check_service_health(service_url) + click.echo(click.style("Resin service is healthy!", fg="green")) + return @cli.command( @@ -313,13 +313,7 @@ def _chat( help="Index name suffix", ) def chat(index_name, chat_service_url, rag, debug, stream): - if not is_healthy(chat_service_url): - msg = ( - f"Resin service is not running! on {chat_service_url}" - + " please run `resin start`" - ) - click.echo(click.style(msg, fg="red"), err=True) - sys.exit(1) + check_service_health(chat_service_url) note_msg = ( "🚨 Note 🚨\n" + "Chat is a debugging tool, it is not meant to be used for production!" @@ -429,13 +423,7 @@ def stop(host, port, ssl): ssl_str = "s" if ssl else "" service_url = f"http{ssl_str}://{host}:{port}" - if not is_healthy(service_url): - msg = ( - f"Resin service is not running! on {service_url}" - + " please run `resin start`" - ) - click.echo(click.style(msg, fg="red"), err=True) - sys.exit(1) + check_service_health(service_url) import subprocess From 41bb6926608ac7218a2f39144624cdf6e5d4febe Mon Sep 17 00:00:00 2001 From: ilai Date: Tue, 17 Oct 2023 14:54:34 +0300 Subject: [PATCH 13/64] [cli] Improve health check errors Give more detailed indication of the actual underlying problem. --- src/resin/llm/openai.py | 4 ++-- src/resin_cli/cli.py | 18 ++++++++---------- src/resin_cli/data_loader/__init__.py | 1 + src/resin_cli/data_loader/data_loader.py | 15 +++++++++++---- 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/src/resin/llm/openai.py b/src/resin/llm/openai.py index d9ea5f96..af7ca672 100644 --- a/src/resin/llm/openai.py +++ b/src/resin/llm/openai.py @@ -27,8 +27,8 @@ def __init__(self, self.available_models = [k["id"] for k in openai.Model.list().data] if model_name not in self.available_models: raise ValueError( - f"Model {model_name} not found. " + - " Available models: {self.available_models}" + f"Model {model_name} not found. " + f" Available models: {self.available_models}" ) @retry( diff --git a/src/resin_cli/cli.py b/src/resin_cli/cli.py index 4d35aa50..5d3fef98 100644 --- a/src/resin_cli/cli.py +++ b/src/resin_cli/cli.py @@ -1,5 +1,4 @@ import os -from textwrap import dedent import click import time @@ -18,6 +17,7 @@ from resin.tokenizer import OpenAITokenizer, Tokenizer from resin_cli.data_loader import ( load_from_path, + CLIError, IDsNotUniqueError, DocumentsValidationError) @@ -32,7 +32,6 @@ load_dotenv(dotenv_path) spinner = Spinner() -format_multiline = lambda s: dedent(s).strip() def check_service_health(url: str): @@ -44,17 +43,16 @@ def check_service_health(url: str): except requests.exceptions.ConnectionError: msg = f""" Resin service is not running on {url}. - please run `resin start`" + please run `resin start` """ - click.echo(click.style(format_multiline(msg), fg="red"), err=True) - sys.exit(1) - except Exception as e: + raise CLIError(msg) + + except requests.exceptions.HTTPError as e: + error = e.response.json().get("detail", None) or e.response.text msg = ( - f"Resin service on {url} is not healthy, failed with error: {e}" + f"Resin service on {url} is not healthy, failed with error: {error}" ) - click.echo(click.style(msg, fg="red"), err=True) - sys.exit(1) - + raise CLIError(msg) @retry(wait=wait_fixed(5), stop=stop_after_attempt(6)) def wait_for_service(chat_service_url: str): diff --git a/src/resin_cli/data_loader/__init__.py b/src/resin_cli/data_loader/__init__.py index 8a298030..85464408 100644 --- a/src/resin_cli/data_loader/__init__.py +++ b/src/resin_cli/data_loader/__init__.py @@ -1,5 +1,6 @@ from .data_loader import ( load_from_path, + CLIError, IDsNotUniqueError, DocumentsValidationError ) diff --git a/src/resin_cli/data_loader/data_loader.py b/src/resin_cli/data_loader/data_loader.py index 08a40915..7ffcaade 100644 --- a/src/resin_cli/data_loader/data_loader.py +++ b/src/resin_cli/data_loader/data_loader.py @@ -2,8 +2,11 @@ import os import glob from typing import List +from textwrap import dedent +import click import pandas as pd +from click import ClickException from pydantic import ValidationError @@ -11,14 +14,18 @@ class IDsNotUniqueError(ValueError): - def __init__(self, message): - super().__init__(message) + pass class DocumentsValidationError(ValueError): - def __init__(self, message): - super().__init__(message) + pass +format_multiline = lambda s: dedent(s).strip() + + +class CLIError(ClickException): + def format_message(self) -> str: + return click.style(format_multiline(self.message), fg='red') def _process_metadata(value): if pd.isna(value): From 1dde2578c6480e5a30f9de93526222a928926f4a Mon Sep 17 00:00:00 2001 From: ilai Date: Tue, 17 Oct 2023 15:06:50 +0300 Subject: [PATCH 14/64] [cli] Improve connection error Made them a bit more explicit. --- src/resin_cli/cli.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/resin_cli/cli.py b/src/resin_cli/cli.py index 5d3fef98..fb3920d9 100644 --- a/src/resin_cli/cli.py +++ b/src/resin_cli/cli.py @@ -62,23 +62,23 @@ def wait_for_service(chat_service_url: str): def validate_connection(): try: KnowledgeBase._connect_pinecone() - except Exception: + except RuntimeError as e: msg = ( - "Error: Failed to connect to Pinecone index, please make sure" - + " you have set the right env vars" - + " PINECONE_API_KEY, INDEX_NAME, PINECONE_ENVIRONMENT" + f"{str(e)}\n" + "Credentials should be set by the PINECONE_API_KEY and PINECONE_ENVIRONMENT" + " envriorment variables. " + "Please visit https://www.pinecone.io/docs/quick-start/ for more details." ) - click.echo(click.style(msg, fg="red"), err=True) - sys.exit(1) + raise CLIError(msg) try: openai.Model.list() except Exception: msg = ( - "Error: Failed to connect to OpenAI, please make sure" - + " you have set the right env vars" + "Failed to connect to OpenAI, please make sure that the OPENAI_API_KEY " + "environment variable is set correctly.\n" + "Please visit https://platform.openai.com/account/api-keys for more details" ) - click.echo(click.style(msg, fg="red"), err=True) - sys.exit(1) + raise CLIError(msg) click.echo("Resin: ", nl=False) click.echo(click.style("Ready\n", bold=True, fg="green")) From 152b2798b0aded27a5c20ca829e518e9bd8f787e Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Wed, 18 Oct 2023 00:00:49 +0300 Subject: [PATCH 15/64] use BOW values projected to dense as stub embeddings for test --- tests/unit/stubs/stub_dense_encoder.py | 54 ++++++++++++++++++-------- 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/tests/unit/stubs/stub_dense_encoder.py b/tests/unit/stubs/stub_dense_encoder.py index d0e02ff2..8e041dbb 100644 --- a/tests/unit/stubs/stub_dense_encoder.py +++ b/tests/unit/stubs/stub_dense_encoder.py @@ -1,5 +1,7 @@ -import hashlib +import mmh3 import numpy as np +from collections import defaultdict +from scipy.sparse import csr_matrix from typing import Union, List from pinecone_text.dense.base_dense_ecoder import BaseDenseEncoder @@ -7,8 +9,39 @@ class StubDenseEncoder(BaseDenseEncoder): - def __init__(self, dimension: int = 3): + """ + Bag-of-words encoder that uses a random projection matrix to + project sparse vectors to dense vectors. + uses Johnson–Lindenstrauss lemma to project BOW sparse vectors to dense vectors. + """ + + def __init__(self, + dimension: int = 128, + vocab_size: int = 2 ** 20, + seed: int = 42): + self.input_dim = vocab_size self.dimension = dimension + rng = np.random.default_rng(seed) + self.random_matrix = rng.standard_normal((self.input_dim, self.dimension)) + + def _text_to_sparse_vector(self, text: str) -> csr_matrix: + words = text.split() + word_counts = defaultdict(int) + for word in words: + hashed_word = mmh3.hash(word) % self.input_dim + word_counts[hashed_word] += 1 + + indices = list(word_counts.keys()) + values = list(word_counts.values()) + sparse_vector = csr_matrix((values, (np.zeros_like(indices), indices)), + shape=(1, self.input_dim)) + + return sparse_vector + + def _encode_text(self, text: str) -> List[float]: + sparse_vector = self._text_to_sparse_vector(text) + projected_embedding = sparse_vector.dot(self.random_matrix).flatten() + return list(projected_embedding / np.linalg.norm(projected_embedding)) def encode_documents(self, texts: Union[str, List[str]] @@ -20,23 +53,10 @@ def encode_queries(self, ) -> Union[List[float], List[List[float]]]: return self._encode(texts) - def consistent_embedding(self, text: str) -> List[float]: - # consistent embedding function that project each text to a unique angle - embedding = [] - for i in range(self.dimension): - sha256_hash = hashlib.sha256(f"{text} {i}".encode()).hexdigest() - int_value = int(sha256_hash, 16) - embedding.append(int_value / float(1 << 256)) - - l2_norm = np.linalg.norm(embedding) - normalized_embedding = [float(value / l2_norm) for value in embedding] - - return normalized_embedding - def _encode(self, texts: Union[str, List[str]] ) -> Union[List[float], List[List[float]]]: if isinstance(texts, str): - return self.consistent_embedding(texts) + return self._encode_text(texts) else: - return [self.consistent_embedding(text) for text in texts] + return [self._encode_text(text) for text in texts] From 1ba7a36e75ded363cf86d8e0673ae1b537635a96 Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Wed, 18 Oct 2023 10:08:43 +0300 Subject: [PATCH 16/64] add scipy to dev dependencies --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 529c3e87..6cb1a456 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ pytest-mock = "^3.6.1" pytest-xdist = "^3.3.1" types-requests = "^2.31.0.2" httpx = "^0.25.0" +scipy = "^1.7.1" [build-system] requires = ["poetry-core"] From 5e13407c662f66a3dae5fbd5c154ebbc4a2fb412 Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Wed, 18 Oct 2023 10:40:52 +0300 Subject: [PATCH 17/64] use float32 --- tests/unit/stubs/stub_dense_encoder.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/stubs/stub_dense_encoder.py b/tests/unit/stubs/stub_dense_encoder.py index 8e041dbb..6e456919 100644 --- a/tests/unit/stubs/stub_dense_encoder.py +++ b/tests/unit/stubs/stub_dense_encoder.py @@ -41,6 +41,7 @@ def _text_to_sparse_vector(self, text: str) -> csr_matrix: def _encode_text(self, text: str) -> List[float]: sparse_vector = self._text_to_sparse_vector(text) projected_embedding = sparse_vector.dot(self.random_matrix).flatten() + projected_embedding = projected_embedding.astype(np.float32) return list(projected_embedding / np.linalg.norm(projected_embedding)) def encode_documents(self, From a14df188ecd2dabb60f3e4be60b189d3174537ae Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Wed, 18 Oct 2023 12:37:08 +0300 Subject: [PATCH 18/64] reduce default vocab size --- tests/unit/stubs/stub_dense_encoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/stubs/stub_dense_encoder.py b/tests/unit/stubs/stub_dense_encoder.py index 6e456919..2854c31a 100644 --- a/tests/unit/stubs/stub_dense_encoder.py +++ b/tests/unit/stubs/stub_dense_encoder.py @@ -17,7 +17,7 @@ class StubDenseEncoder(BaseDenseEncoder): def __init__(self, dimension: int = 128, - vocab_size: int = 2 ** 20, + vocab_size: int = 2 ** 18, seed: int = 42): self.input_dim = vocab_size self.dimension = dimension From 6a6e3071d9a207ad29dd0e90110d4e324a7ec8ca Mon Sep 17 00:00:00 2001 From: ilai Date: Wed, 18 Oct 2023 15:36:53 +0300 Subject: [PATCH 19/64] [cli] Improve 'new' command error handling + help message - Properly print the error reason in CLI - In kb.create_resin_index() - add underlying error to string - Improved help message --- src/resin/knoweldge_base/knowledge_base.py | 18 ++++++++++----- src/resin_cli/cli.py | 26 +++++++++++++--------- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/src/resin/knoweldge_base/knowledge_base.py b/src/resin/knoweldge_base/knowledge_base.py index 27304805..4b5f8f2b 100644 --- a/src/resin/knoweldge_base/knowledge_base.py +++ b/src/resin/knoweldge_base/knowledge_base.py @@ -6,6 +6,7 @@ import pandas as pd from pinecone import list_indexes, delete_index, create_index, init \ as pinecone_init, whoami as pinecone_whoami +from pinecone import ApiException as PineconeApiException try: from pinecone import GRPCIndex as Index @@ -160,8 +161,15 @@ def create_resin_index(self, "Please remove it from indexed_fields") if dimension is None: - if self._encoder.dimension is not None: - dimension = self._encoder.dimension + try: + encoder_dimension = self._encoder.dimension + except Exception as e: + raise RuntimeError( + f"Failed to infer vectors' dimension from encoder due to error: " + f"{e}. Please fix the error or provide the dimension manually" + ) from e + if encoder_dimension is not None: + dimension = encoder_dimension else: raise ValueError("Could not infer dimension from encoder. " "Please provide the vectors' dimension") @@ -185,10 +193,10 @@ def create_resin_index(self, }, timeout=TIMEOUT_INDEX_CREATE, **index_params) - except Exception as e: + except (Exception, PineconeApiException) as e: raise RuntimeError( - f"Unexpected error while creating index {self.index_name}." - f"Please try again." + f"Failed to create index {self.index_name} due to error: " + f"{e.body if isinstance(e, PineconeApiException) else e}" ) from e # wait for index to be provisioned diff --git a/src/resin_cli/cli.py b/src/resin_cli/cli.py index fb3920d9..46feb268 100644 --- a/src/resin_cli/cli.py +++ b/src/resin_cli/cli.py @@ -30,6 +30,8 @@ dotenv_path = os.path.join(os.path.dirname(__file__), ".env") load_dotenv(dotenv_path) +if os.getenv("OPENAI_API_KEY"): + openai.api_key = os.getenv("OPENAI_API_KEY") spinner = Spinner() @@ -112,29 +114,31 @@ def health(host, port, ssl): @cli.command( help=( - "New command sets up a new index in Pinecone that is configured for Resin." - + " This will automatically tap the embedding model with a single toeken to " - + "assert for the dimensionality of the embedding space. This will also set up " - + "the index with the right schema for Resin." + """Create a new Pinecone index that that will be used by Resin. + + A Resin webapp can not be started without a Pinecone index which is configured + to work with Resin. + This command will create a new Pinecone index and configure it in the right + schema for Resin. If the embedding's dimension is not explicitly configured by + the config file - the embedding model will be tapped with a single token to + infer the dimensionality of the embedding space. + """ ) ) @click.argument("index-name", nargs=1, envvar="INDEX_NAME", type=str, required=True) -@click.option("--tokenizer-model", default="gpt-3.5-turbo", help="Tokenizer model") -def new(index_name, tokenizer_model): +def new(index_name): + Tokenizer.initialize() kb = KnowledgeBase(index_name=index_name) click.echo("Resin is going to create a new index: ", nl=False) click.echo(click.style(f"{kb.index_name}", fg="green")) click.confirm(click.style("Do you want to continue?", fg="red"), abort=True) - Tokenizer.initialize(OpenAITokenizer, tokenizer_model) with spinner: try: kb.create_resin_index() # TODO: kb should throw a specific exception for each case except Exception as e: - msg = "Error: Failed to create a new index" - click.echo(click.style(msg, fg="red"), err=True, nl=False) - click.echo(f" Reason: {e}") - sys.exit(1) + msg = f"Failed to create a new index. Reason:\n{e}" + raise CLIError(msg) click.echo(click.style("Success!", fg="green")) os.environ["INDEX_NAME"] = index_name From 6bda91d78a90e3d51fe9d83ece4ed07b156ea8ee Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Wed, 18 Oct 2023 17:13:25 +0300 Subject: [PATCH 20/64] remove the need in random matrix --- pyproject.toml | 1 - .../knowledge_base/test_knowledge_base.py | 2 +- tests/unit/stubs/stub_dense_encoder.py | 30 +++++++++---------- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6cb1a456..529c3e87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,6 @@ pytest-mock = "^3.6.1" pytest-xdist = "^3.3.1" types-requests = "^2.31.0.2" httpx = "^0.25.0" -scipy = "^1.7.1" [build-system] requires = ["poetry-core"] diff --git a/tests/system/knowledge_base/test_knowledge_base.py b/tests/system/knowledge_base/test_knowledge_base.py index 3ede466a..0f995ef5 100644 --- a/tests/system/knowledge_base/test_knowledge_base.py +++ b/tests/system/knowledge_base/test_knowledge_base.py @@ -59,7 +59,7 @@ def chunker(): @pytest.fixture(scope="module") def encoder(): return StubRecordEncoder( - StubDenseEncoder(dimension=3)) + StubDenseEncoder()) @pytest.fixture(scope="module", autouse=True) diff --git a/tests/unit/stubs/stub_dense_encoder.py b/tests/unit/stubs/stub_dense_encoder.py index 2854c31a..9d55bb7f 100644 --- a/tests/unit/stubs/stub_dense_encoder.py +++ b/tests/unit/stubs/stub_dense_encoder.py @@ -1,14 +1,12 @@ import mmh3 import numpy as np from collections import defaultdict -from scipy.sparse import csr_matrix from typing import Union, List from pinecone_text.dense.base_dense_ecoder import BaseDenseEncoder class StubDenseEncoder(BaseDenseEncoder): - """ Bag-of-words encoder that uses a random projection matrix to project sparse vectors to dense vectors. @@ -16,31 +14,31 @@ class StubDenseEncoder(BaseDenseEncoder): """ def __init__(self, - dimension: int = 128, - vocab_size: int = 2 ** 18, - seed: int = 42): + dimension: int = 8, + vocab_size: int = 2 ** 12): self.input_dim = vocab_size self.dimension = dimension - rng = np.random.default_rng(seed) - self.random_matrix = rng.standard_normal((self.input_dim, self.dimension)) - def _text_to_sparse_vector(self, text: str) -> csr_matrix: + def _text_to_word_counts(self, text: str) -> defaultdict: words = text.split() word_counts = defaultdict(int) for word in words: hashed_word = mmh3.hash(word) % self.input_dim word_counts[hashed_word] += 1 + return word_counts + + def _encode_text(self, text: str) -> List[float]: + word_counts = self._text_to_word_counts(text) - indices = list(word_counts.keys()) - values = list(word_counts.values()) - sparse_vector = csr_matrix((values, (np.zeros_like(indices), indices)), - shape=(1, self.input_dim)) + # This will hold the result of word_counts * random_matrix + projected_embedding = np.zeros(self.dimension, dtype=np.float32) - return sparse_vector + for hashed_word, count in word_counts.items(): + rng = np.random.default_rng(hashed_word) + # Seed the RNG with the hashed word index for consistency + random_vector = rng.standard_normal(self.dimension) + projected_embedding += count * random_vector - def _encode_text(self, text: str) -> List[float]: - sparse_vector = self._text_to_sparse_vector(text) - projected_embedding = sparse_vector.dot(self.random_matrix).flatten() projected_embedding = projected_embedding.astype(np.float32) return list(projected_embedding / np.linalg.norm(projected_embedding)) From c1f200698da0016b155770954345ff9653ce3b80 Mon Sep 17 00:00:00 2001 From: ilai Date: Wed, 18 Oct 2023 17:42:51 +0300 Subject: [PATCH 21/64] [cli] Improve upsert error messages There is still a big gap, which is the data validation error handling. Would fix that in a separate PR later --- src/resin_cli/cli.py | 100 ++++++++++++++++++++----------------------- 1 file changed, 46 insertions(+), 54 deletions(-) diff --git a/src/resin_cli/cli.py b/src/resin_cli/cli.py index 46feb268..f585d476 100644 --- a/src/resin_cli/cli.py +++ b/src/resin_cli/cli.py @@ -34,6 +34,7 @@ openai.api_key = os.getenv("OPENAI_API_KEY") spinner = Spinner() +CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) def check_service_health(url: str): @@ -68,7 +69,7 @@ def validate_connection(): msg = ( f"{str(e)}\n" "Credentials should be set by the PINECONE_API_KEY and PINECONE_ENVIRONMENT" - " envriorment variables. " + " environment variables. " "Please visit https://www.pinecone.io/docs/quick-start/ for more details." ) raise CLIError(msg) @@ -85,7 +86,15 @@ def validate_connection(): click.echo(click.style("Ready\n", bold=True, fg="green")) -@click.group(invoke_without_command=True) +def _initialize_tokenizer(): + try: + Tokenizer.initialize() + except Exception as e: + msg = f"Failed to initialize tokenizer. Reason:\n{e}" + raise CLIError(msg) + + +@click.group(invoke_without_command=True, context_settings=CONTEXT_SETTINGS) @click.version_option(__version__, "-v", "--version", prog_name="Resin") @click.pass_context def cli(ctx): @@ -99,7 +108,7 @@ def cli(ctx): click.echo(ctx.get_help()) -@cli.command(help="Check if resin service is running by sending a health check request") +@cli.command(help="Check if resin service is running and healthy.") @click.option("--host", default="0.0.0.0", help="Resin's service hostname") @click.option("--port", default=8000, help="The port of the resin service") @click.option("--ssl/--no-ssl", default=False, help="Whether to use ssl for the " @@ -116,7 +125,7 @@ def health(host, port, ssl): help=( """Create a new Pinecone index that that will be used by Resin. - A Resin webapp can not be started without a Pinecone index which is configured + A Resin service can not be started without a Pinecone index which is configured to work with Resin. This command will create a new Pinecone index and configure it in the right schema for Resin. If the embedding's dimension is not explicitly configured by @@ -127,7 +136,7 @@ def health(host, port, ssl): ) @click.argument("index-name", nargs=1, envvar="INDEX_NAME", type=str, required=True) def new(index_name): - Tokenizer.initialize() + _initialize_tokenizer() kb = KnowledgeBase(index_name=index_name) click.echo("Resin is going to create a new index: ", nl=False) click.echo(click.style(f"{kb.index_name}", fg="green")) @@ -145,47 +154,40 @@ def new(index_name): @cli.command( help=( - "Upsert allows you to load a loacal data file into a your Resin index." - + " The allowed formats are .jsonl and .parquet. The data will be validated" + """Upload local data files containing documents to the Resin service. + + Load all the documents from data file or a directory containing multiple data + files. The allowed formats are .jsonl and .parquet. + """ ) ) @click.argument("data-path", type=click.Path(exists=True)) @click.option( "--index-name", default=os.environ.get("INDEX_NAME"), - help="Index name", + help="The name of the index to upload the data to. " + "Inferred from INDEX_NAME env var if not provided." ) -@click.option("--tokenizer-model", default="gpt-3.5-turbo", help="Tokenizer model") -def upsert(index_name, data_path, tokenizer_model): +def upsert(index_name, data_path): if index_name is None: msg = ("Index name is not provided, please provide it with" + ' --index-name or set it with env var + ' '`export INDEX_NAME="MY_INDEX_NAME`') - click.echo(click.style(msg, fg="red"), err=True) - sys.exit(1) - try: - Tokenizer.initialize(OpenAITokenizer, tokenizer_model) - except Exception: - msg = "Error: Failed to initialize tokenizer" - click.echo(click.style(msg, fg="red"), err=True) - sys.exit(1) - if data_path is None: - msg = ("Data path is not provided," - " please provide it with --data-path or set it with env var") - click.echo(click.style(msg, fg="red"), err=True) - sys.exit(1) + raise CLIError(msg) + + _initialize_tokenizer() kb = KnowledgeBase(index_name=index_name) try: kb.connect() - except Exception: - msg = ( - "Error: Failed to connect to Pinecone index, please make sure" - " you have set the right env vars" - " PINECONE_API_KEY, INDEX_NAME, PINECONE_ENVIRONMENT" - ) - click.echo(click.style(msg, fg="red"), err=True) - sys.exit(1) + except RuntimeError as e: + # TODO: kb should throw a specific exception for each case + msg = str(e) + if "credentials" in msg: + msg += ("\nCredentials should be set by the PINECONE_API_KEY and " + "PINECONE_ENVIRONMENT environment variables. Please visit " + "https://www.pinecone.io/docs/quick-start/ for more details.") + raise CLIError(msg) click.echo("Resin is going to upsert data from ", nl=False) click.echo(click.style(f"{data_path}", fg="yellow"), nl=False) @@ -196,43 +198,33 @@ def upsert(index_name, data_path, tokenizer_model): data = load_from_path(data_path) except IDsNotUniqueError: msg = ( - "Error: the id field on the data is not unique" - + " this will cause records to override each other on upsert" - + " please make sure the id field is unique" + "The data contains duplicate IDs, please make sure that each document" + " has a unique ID, otherwise documents with the same ID will overwrite" + " each other" ) - click.echo(click.style(msg, fg="red"), err=True) - sys.exit(1) + raise CLIError(msg) except DocumentsValidationError: msg = ( - "Error: one or more rows have not passed validation" - + " data should agree with the Document Schema" - + f" on {Document.__annotations__}" - + " please make sure the data is valid" + f"One or more rows have failed data validation. The rows in the" + f"data file should be in the schema: {Document.__annotations__}." ) - click.echo(click.style(msg, fg="red"), err=True) - sys.exit(1) + raise CLIError(msg) except Exception: msg = ( - "Error: an unexpected error has occured in loading data from files" - + " it may be due to issue with the data format" - + " please make sure the data is valid, and can load with pandas" + f"A unexpected error while loading the data from files in {data_path}. " + "Please make sure the data is in valid `jsonl` or `parquet` format." ) - click.echo(click.style(msg, fg="red"), err=True) - sys.exit(1) + raise CLIError(msg) pd.options.display.max_colwidth = 20 click.echo(data[0].json(exclude_none=True, indent=2)) click.confirm(click.style("\nDoes this data look right?", fg="red"), abort=True) try: kb.upsert(data) - except Exception: + except Exception as e: msg = ( - "Error: Failed to upsert data to index" - f" {kb.index_name}" - " this could be due to connection issues" - " please re-run `resin upsert`" + f"Failed to upsert data to index {kb.index_name}. Underlying error: {e}" ) - click.echo(click.style(msg, fg="red"), err=True) - sys.exit(1) + raise CLIError(msg) click.echo(click.style("Success!", fg="green")) From 88fa65842dfd0a4465271b5a1772f65646dd899d Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Thu, 19 Oct 2023 10:50:35 +0300 Subject: [PATCH 22/64] fix type in kb name --- .../context_engine/context_builder/base.py | 2 +- .../context_builder/stuffing.py | 2 +- src/resin/context_engine/context_engine.py | 4 ++-- .../__init__.py | 0 .../base.py | 2 +- .../chunker/__init__.py | 0 .../chunker/base.py | 2 +- .../chunker/langchain_text_splitter.py | 0 .../chunker/markdown.py | 2 +- .../chunker/recursive_character.py | 4 ++-- .../chunker/token_chunker.py | 0 .../knowledge_base.py | 20 +++++++++---------- .../models.py | 0 .../record_encoder/__init__.py | 0 .../record_encoder/base.py | 2 +- .../record_encoder/dense.py | 2 +- .../record_encoder/openai.py | 4 ++-- .../reranker/__init__.py | 0 .../reranker/reranker.py | 2 +- src/resin_cli/app.py | 2 +- src/resin_cli/cli.py | 2 +- tests/e2e/test_app.py | 2 +- .../knowledge_base/test_knowledge_base.py | 12 +++++------ tests/unit/chunker/test_markdown_chunker.py | 4 ++-- .../test_recursive_character_chunker.py | 4 ++-- tests/unit/chunker/test_stub_chunker.py | 2 +- tests/unit/chunker/test_token_chunker.py | 4 ++-- .../test_stuffing_context_builder.py | 2 +- .../context_engine/test_context_engine.py | 4 ++-- .../base_test_record_encoder.py | 2 +- .../test_dense_record_encoder.py | 2 +- .../test_openai_record_encoder.py | 2 +- tests/unit/stubs/stub_chunker.py | 4 ++-- tests/unit/stubs/stub_record_encoder.py | 4 ++-- 34 files changed, 50 insertions(+), 50 deletions(-) rename src/resin/{knoweldge_base => knowledge_base}/__init__.py (100%) rename src/resin/{knoweldge_base => knowledge_base}/base.py (96%) rename src/resin/{knoweldge_base => knowledge_base}/chunker/__init__.py (100%) rename src/resin/{knoweldge_base => knowledge_base}/chunker/base.py (95%) rename src/resin/{knoweldge_base => knowledge_base}/chunker/langchain_text_splitter.py (100%) rename src/resin/{knoweldge_base => knowledge_base}/chunker/markdown.py (94%) rename src/resin/{knoweldge_base => knowledge_base}/chunker/recursive_character.py (92%) rename src/resin/{knoweldge_base => knowledge_base}/chunker/token_chunker.py (100%) rename src/resin/{knoweldge_base => knowledge_base}/knowledge_base.py (97%) rename src/resin/{knoweldge_base => knowledge_base}/models.py (100%) rename src/resin/{knoweldge_base => knowledge_base}/record_encoder/__init__.py (100%) rename src/resin/{knoweldge_base => knowledge_base}/record_encoder/base.py (97%) rename src/resin/{knoweldge_base => knowledge_base}/record_encoder/dense.py (95%) rename src/resin/{knoweldge_base => knowledge_base}/record_encoder/openai.py (91%) rename src/resin/{knoweldge_base => knowledge_base}/reranker/__init__.py (100%) rename src/resin/{knoweldge_base => knowledge_base}/reranker/reranker.py (91%) diff --git a/src/resin/context_engine/context_builder/base.py b/src/resin/context_engine/context_builder/base.py index 81c1d281..84f5283a 100644 --- a/src/resin/context_engine/context_builder/base.py +++ b/src/resin/context_engine/context_builder/base.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from typing import List -from resin.knoweldge_base.models import QueryResult +from resin.knowledge_base.models import QueryResult from resin.models.data_models import Context from resin.utils.config import ConfigurableMixin diff --git a/src/resin/context_engine/context_builder/stuffing.py b/src/resin/context_engine/context_builder/stuffing.py index 4bed0cef..c7ebb70e 100644 --- a/src/resin/context_engine/context_builder/stuffing.py +++ b/src/resin/context_engine/context_builder/stuffing.py @@ -3,7 +3,7 @@ from resin.context_engine.context_builder.base import ContextBuilder from resin.context_engine.models import ContextQueryResult, ContextSnippet -from resin.knoweldge_base.models import QueryResult, DocumentWithScore +from resin.knowledge_base.models import QueryResult, DocumentWithScore from resin.tokenizer import Tokenizer from resin.models.data_models import Context diff --git a/src/resin/context_engine/context_engine.py b/src/resin/context_engine/context_engine.py index 1507e926..ae99c46c 100644 --- a/src/resin/context_engine/context_engine.py +++ b/src/resin/context_engine/context_engine.py @@ -4,8 +4,8 @@ from resin.context_engine.context_builder import StuffingContextBuilder from resin.context_engine.context_builder.base import ContextBuilder -from resin.knoweldge_base import KnowledgeBase -from resin.knoweldge_base.base import BaseKnowledgeBase +from resin.knowledge_base import KnowledgeBase +from resin.knowledge_base.base import BaseKnowledgeBase from resin.models.data_models import Context, Query from resin.utils.config import ConfigurableMixin diff --git a/src/resin/knoweldge_base/__init__.py b/src/resin/knowledge_base/__init__.py similarity index 100% rename from src/resin/knoweldge_base/__init__.py rename to src/resin/knowledge_base/__init__.py diff --git a/src/resin/knoweldge_base/base.py b/src/resin/knowledge_base/base.py similarity index 96% rename from src/resin/knoweldge_base/base.py rename to src/resin/knowledge_base/base.py index 32ca4152..f639b8a5 100644 --- a/src/resin/knoweldge_base/base.py +++ b/src/resin/knowledge_base/base.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from typing import List, Optional -from resin.knoweldge_base.models import QueryResult +from resin.knowledge_base.models import QueryResult from resin.models.data_models import Query, Document from resin.utils.config import ConfigurableMixin diff --git a/src/resin/knoweldge_base/chunker/__init__.py b/src/resin/knowledge_base/chunker/__init__.py similarity index 100% rename from src/resin/knoweldge_base/chunker/__init__.py rename to src/resin/knowledge_base/chunker/__init__.py diff --git a/src/resin/knoweldge_base/chunker/base.py b/src/resin/knowledge_base/chunker/base.py similarity index 95% rename from src/resin/knoweldge_base/chunker/base.py rename to src/resin/knowledge_base/chunker/base.py index a0ccee5a..a375073d 100644 --- a/src/resin/knoweldge_base/chunker/base.py +++ b/src/resin/knowledge_base/chunker/base.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from typing import List -from resin.knoweldge_base.models import KBDocChunk +from resin.knowledge_base.models import KBDocChunk from resin.models.data_models import Document from resin.utils.config import ConfigurableMixin diff --git a/src/resin/knoweldge_base/chunker/langchain_text_splitter.py b/src/resin/knowledge_base/chunker/langchain_text_splitter.py similarity index 100% rename from src/resin/knoweldge_base/chunker/langchain_text_splitter.py rename to src/resin/knowledge_base/chunker/langchain_text_splitter.py diff --git a/src/resin/knoweldge_base/chunker/markdown.py b/src/resin/knowledge_base/chunker/markdown.py similarity index 94% rename from src/resin/knoweldge_base/chunker/markdown.py rename to src/resin/knowledge_base/chunker/markdown.py index cedf2bc2..ad8976c3 100644 --- a/src/resin/knoweldge_base/chunker/markdown.py +++ b/src/resin/knowledge_base/chunker/markdown.py @@ -2,7 +2,7 @@ from .langchain_text_splitter import Language, RecursiveCharacterTextSplitter from .recursive_character import RecursiveCharacterChunker -from resin.knoweldge_base.models import KBDocChunk +from resin.knowledge_base.models import KBDocChunk from resin.models.data_models import Document diff --git a/src/resin/knoweldge_base/chunker/recursive_character.py b/src/resin/knowledge_base/chunker/recursive_character.py similarity index 92% rename from src/resin/knoweldge_base/chunker/recursive_character.py rename to src/resin/knowledge_base/chunker/recursive_character.py index ab942355..73651fd0 100644 --- a/src/resin/knoweldge_base/chunker/recursive_character.py +++ b/src/resin/knowledge_base/chunker/recursive_character.py @@ -3,8 +3,8 @@ from .langchain_text_splitter import RecursiveCharacterTextSplitter -from resin.knoweldge_base.chunker.base import Chunker -from resin.knoweldge_base.models import KBDocChunk +from resin.knowledge_base.chunker.base import Chunker +from resin.knowledge_base.models import KBDocChunk from resin.tokenizer import Tokenizer from resin.models.data_models import Document diff --git a/src/resin/knoweldge_base/chunker/token_chunker.py b/src/resin/knowledge_base/chunker/token_chunker.py similarity index 100% rename from src/resin/knoweldge_base/chunker/token_chunker.py rename to src/resin/knowledge_base/chunker/token_chunker.py diff --git a/src/resin/knoweldge_base/knowledge_base.py b/src/resin/knowledge_base/knowledge_base.py similarity index 97% rename from src/resin/knoweldge_base/knowledge_base.py rename to src/resin/knowledge_base/knowledge_base.py index c0871341..05f8fdb6 100644 --- a/src/resin/knoweldge_base/knowledge_base.py +++ b/src/resin/knowledge_base/knowledge_base.py @@ -15,13 +15,13 @@ from pinecone_datasets import Dataset from pinecone_datasets import DenseModelMetadata, DatasetMetadata -from resin.knoweldge_base.base import BaseKnowledgeBase -from resin.knoweldge_base.chunker import Chunker, MarkdownChunker -from resin.knoweldge_base.record_encoder import (RecordEncoder, +from resin.knowledge_base.base import BaseKnowledgeBase +from resin.knowledge_base.chunker import Chunker, MarkdownChunker +from resin.knowledge_base.record_encoder import (RecordEncoder, OpenAIRecordEncoder) -from resin.knoweldge_base.models import (KBQueryResult, KBQuery, QueryResult, +from resin.knowledge_base.models import (KBQueryResult, KBQuery, QueryResult, KBDocChunkWithScore, DocumentWithScore) -from resin.knoweldge_base.reranker import Reranker, TransparentReranker +from resin.knowledge_base.reranker import Reranker, TransparentReranker from resin.models.data_models import Query, Document @@ -52,7 +52,7 @@ class KnowledgeBase(BaseKnowledgeBase): This is a one-time setup process - the index will exist on Pinecone's managed service until it is deleted. Example: - >>> from resin.knoweldge_base.knowledge_base import KnowledgeBase + >>> from resin.knowledge_base.knowledge_base import KnowledgeBase >>> from tokenizer import Tokenizer >>> Tokenizer.initialize() >>> kb = KnowledgeBase(index_name="my_index") @@ -89,7 +89,7 @@ def __init__(self, Example: create a new index: - >>> from resin.knoweldge_base.knowledge_base import KnowledgeBase + >>> from resin.knowledge_base.knowledge_base import KnowledgeBase >>> from tokenizer import Tokenizer >>> Tokenizer.initialize() >>> kb = KnowledgeBase(index_name="my_index") @@ -387,7 +387,7 @@ def query(self, A list of QueryResult objects. Examples: - >>> from resin.knoweldge_base.knowledge_base import KnowledgeBase + >>> from resin.knowledge_base.knowledge_base import KnowledgeBase >>> from tokenizer import Tokenizer >>> Tokenizer.initialize() >>> kb = KnowledgeBase(index_name="my_index") @@ -480,7 +480,7 @@ def upsert(self, None Example: - >>> from resin.knoweldge_base.knowledge_base import KnowledgeBase + >>> from resin.knowledge_base.knowledge_base import KnowledgeBase >>> from tokenizer import Tokenizer >>> Tokenizer.initialize() >>> kb = KnowledgeBase(index_name="my_index") @@ -558,7 +558,7 @@ def delete(self, None Example: - >>> from resin.knoweldge_base.knowledge_base import KnowledgeBase + >>> from resin.knowledge_base.knowledge_base import KnowledgeBase >>> from tokenizer import Tokenizer >>> Tokenizer.initialize() >>> kb = KnowledgeBase(index_name="my_index") diff --git a/src/resin/knoweldge_base/models.py b/src/resin/knowledge_base/models.py similarity index 100% rename from src/resin/knoweldge_base/models.py rename to src/resin/knowledge_base/models.py diff --git a/src/resin/knoweldge_base/record_encoder/__init__.py b/src/resin/knowledge_base/record_encoder/__init__.py similarity index 100% rename from src/resin/knoweldge_base/record_encoder/__init__.py rename to src/resin/knowledge_base/record_encoder/__init__.py diff --git a/src/resin/knoweldge_base/record_encoder/base.py b/src/resin/knowledge_base/record_encoder/base.py similarity index 97% rename from src/resin/knoweldge_base/record_encoder/base.py rename to src/resin/knowledge_base/record_encoder/base.py index 98db5e8f..47284781 100644 --- a/src/resin/knoweldge_base/record_encoder/base.py +++ b/src/resin/knowledge_base/record_encoder/base.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from typing import List, Optional -from resin.knoweldge_base.models import KBEncodedDocChunk, KBQuery, KBDocChunk +from resin.knowledge_base.models import KBEncodedDocChunk, KBQuery, KBDocChunk from resin.models.data_models import Query from resin.utils.config import ConfigurableMixin diff --git a/src/resin/knoweldge_base/record_encoder/dense.py b/src/resin/knowledge_base/record_encoder/dense.py similarity index 95% rename from src/resin/knoweldge_base/record_encoder/dense.py rename to src/resin/knowledge_base/record_encoder/dense.py index 77a10aeb..8db12001 100644 --- a/src/resin/knoweldge_base/record_encoder/dense.py +++ b/src/resin/knowledge_base/record_encoder/dense.py @@ -3,7 +3,7 @@ from pinecone_text.dense.base_dense_ecoder import BaseDenseEncoder from .base import RecordEncoder -from resin.knoweldge_base.models import KBQuery, KBEncodedDocChunk, KBDocChunk +from resin.knowledge_base.models import KBQuery, KBEncodedDocChunk, KBDocChunk from resin.models.data_models import Query diff --git a/src/resin/knoweldge_base/record_encoder/openai.py b/src/resin/knowledge_base/record_encoder/openai.py similarity index 91% rename from src/resin/knoweldge_base/record_encoder/openai.py rename to src/resin/knowledge_base/record_encoder/openai.py index d42df14f..b56cb743 100644 --- a/src/resin/knoweldge_base/record_encoder/openai.py +++ b/src/resin/knowledge_base/record_encoder/openai.py @@ -6,8 +6,8 @@ retry_if_exception_type, ) from pinecone_text.dense.openai_encoder import OpenAIEncoder -from resin.knoweldge_base.models import KBDocChunk, KBEncodedDocChunk, KBQuery -from resin.knoweldge_base.record_encoder.dense import DenseRecordEncoder +from resin.knowledge_base.models import KBDocChunk, KBEncodedDocChunk, KBQuery +from resin.knowledge_base.record_encoder.dense import DenseRecordEncoder from resin.models.data_models import Query from resin.utils.openai_exceptions import OPEN_AI_TRANSIENT_EXCEPTIONS diff --git a/src/resin/knoweldge_base/reranker/__init__.py b/src/resin/knowledge_base/reranker/__init__.py similarity index 100% rename from src/resin/knoweldge_base/reranker/__init__.py rename to src/resin/knowledge_base/reranker/__init__.py diff --git a/src/resin/knoweldge_base/reranker/reranker.py b/src/resin/knowledge_base/reranker/reranker.py similarity index 91% rename from src/resin/knoweldge_base/reranker/reranker.py rename to src/resin/knowledge_base/reranker/reranker.py index 3ffc9554..04fe9f1f 100644 --- a/src/resin/knoweldge_base/reranker/reranker.py +++ b/src/resin/knowledge_base/reranker/reranker.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from typing import List -from resin.knoweldge_base.models import KBQueryResult +from resin.knowledge_base.models import KBQueryResult from resin.utils.config import ConfigurableMixin diff --git a/src/resin_cli/app.py b/src/resin_cli/app.py index ab1c743f..300935e2 100644 --- a/src/resin_cli/app.py +++ b/src/resin_cli/app.py @@ -7,7 +7,7 @@ from resin.llm import BaseLLM from resin.llm.models import UserMessage from resin.tokenizer import OpenAITokenizer, Tokenizer -from resin.knoweldge_base import KnowledgeBase +from resin.knowledge_base import KnowledgeBase from resin.context_engine import ContextEngine from resin.chat_engine import ChatEngine from starlette.concurrency import run_in_threadpool diff --git a/src/resin_cli/cli.py b/src/resin_cli/cli.py index 04796c6b..587bc7af 100644 --- a/src/resin_cli/cli.py +++ b/src/resin_cli/cli.py @@ -10,7 +10,7 @@ import pandas as pd import openai -from resin.knoweldge_base import KnowledgeBase +from resin.knowledge_base import KnowledgeBase from resin.models.data_models import Document from resin.tokenizer import OpenAITokenizer, Tokenizer from resin_cli.data_loader import ( diff --git a/tests/e2e/test_app.py b/tests/e2e/test_app.py index 5cb50599..19a16b06 100644 --- a/tests/e2e/test_app.py +++ b/tests/e2e/test_app.py @@ -8,7 +8,7 @@ from fastapi.testclient import TestClient from tenacity import retry, stop_after_attempt, wait_fixed -from resin.knoweldge_base import KnowledgeBase +from resin.knowledge_base import KnowledgeBase from resin_cli.app import app from resin_cli.api_models import HealthStatus, ContextUpsertRequest, ContextQueryRequest diff --git a/tests/system/knowledge_base/test_knowledge_base.py b/tests/system/knowledge_base/test_knowledge_base.py index 3ede466a..7963e73a 100644 --- a/tests/system/knowledge_base/test_knowledge_base.py +++ b/tests/system/knowledge_base/test_knowledge_base.py @@ -12,12 +12,12 @@ ) from dotenv import load_dotenv from datetime import datetime -from resin.knoweldge_base import KnowledgeBase -from resin.knoweldge_base.chunker import Chunker -from resin.knoweldge_base.knowledge_base import INDEX_NAME_PREFIX -from resin.knoweldge_base.models import DocumentWithScore -from resin.knoweldge_base.record_encoder import RecordEncoder -from resin.knoweldge_base.reranker import Reranker +from resin.knowledge_base import KnowledgeBase +from resin.knowledge_base.chunker import Chunker +from resin.knowledge_base.knowledge_base import INDEX_NAME_PREFIX +from resin.knowledge_base.models import DocumentWithScore +from resin.knowledge_base.record_encoder import RecordEncoder +from resin.knowledge_base.reranker import Reranker from resin.models.data_models import Document, Query from tests.unit.stubs.stub_record_encoder import StubRecordEncoder from tests.unit.stubs.stub_dense_encoder import StubDenseEncoder diff --git a/tests/unit/chunker/test_markdown_chunker.py b/tests/unit/chunker/test_markdown_chunker.py index 9231b7b1..ada9b2a3 100644 --- a/tests/unit/chunker/test_markdown_chunker.py +++ b/tests/unit/chunker/test_markdown_chunker.py @@ -1,7 +1,7 @@ import pytest -from resin.knoweldge_base.chunker import MarkdownChunker -from resin.knoweldge_base.models import KBDocChunk +from resin.knowledge_base.chunker import MarkdownChunker +from resin.knowledge_base.models import KBDocChunk from resin.models.data_models import Document from tests.unit.chunker.base_test_chunker import BaseTestChunker diff --git a/tests/unit/chunker/test_recursive_character_chunker.py b/tests/unit/chunker/test_recursive_character_chunker.py index 282e4875..3a3cc8fe 100644 --- a/tests/unit/chunker/test_recursive_character_chunker.py +++ b/tests/unit/chunker/test_recursive_character_chunker.py @@ -1,7 +1,7 @@ import pytest -from resin.knoweldge_base.chunker.recursive_character \ +from resin.knowledge_base.chunker.recursive_character \ import RecursiveCharacterChunker -from resin.knoweldge_base.models import KBDocChunk +from resin.knowledge_base.models import KBDocChunk from tests.unit.chunker.base_test_chunker import BaseTestChunker diff --git a/tests/unit/chunker/test_stub_chunker.py b/tests/unit/chunker/test_stub_chunker.py index bb84e418..78eb91a3 100644 --- a/tests/unit/chunker/test_stub_chunker.py +++ b/tests/unit/chunker/test_stub_chunker.py @@ -1,6 +1,6 @@ import pytest -from resin.knoweldge_base.models import KBDocChunk +from resin.knowledge_base.models import KBDocChunk from .base_test_chunker import BaseTestChunker from ..stubs.stub_chunker import StubChunker diff --git a/tests/unit/chunker/test_token_chunker.py b/tests/unit/chunker/test_token_chunker.py index b0de1040..463dc99f 100644 --- a/tests/unit/chunker/test_token_chunker.py +++ b/tests/unit/chunker/test_token_chunker.py @@ -1,9 +1,9 @@ import pytest -from resin.knoweldge_base.models import KBDocChunk +from resin.knowledge_base.models import KBDocChunk from resin.models.data_models import Document from .base_test_chunker import BaseTestChunker -from resin.knoweldge_base.chunker.token_chunker import TokenChunker +from resin.knowledge_base.chunker.token_chunker import TokenChunker class TestTokenChunker(BaseTestChunker): diff --git a/tests/unit/context_builder/test_stuffing_context_builder.py b/tests/unit/context_builder/test_stuffing_context_builder.py index 3940df8b..f8ec3220 100644 --- a/tests/unit/context_builder/test_stuffing_context_builder.py +++ b/tests/unit/context_builder/test_stuffing_context_builder.py @@ -2,7 +2,7 @@ ContextSnippet, ContextQueryResult from resin.models.data_models import Context from ..stubs.stub_tokenizer import StubTokenizer -from resin.knoweldge_base.models import \ +from resin.knowledge_base.models import \ QueryResult, DocumentWithScore from resin.context_engine.context_builder import StuffingContextBuilder diff --git a/tests/unit/context_engine/test_context_engine.py b/tests/unit/context_engine/test_context_engine.py index 10255b93..41e87079 100644 --- a/tests/unit/context_engine/test_context_engine.py +++ b/tests/unit/context_engine/test_context_engine.py @@ -3,8 +3,8 @@ from resin.context_engine import ContextEngine from resin.context_engine.context_builder.base import ContextBuilder -from resin.knoweldge_base.base import BaseKnowledgeBase -from resin.knoweldge_base.models import QueryResult, DocumentWithScore +from resin.knowledge_base.base import BaseKnowledgeBase +from resin.knowledge_base.models import QueryResult, DocumentWithScore from resin.models.data_models import Query, Context, ContextContent diff --git a/tests/unit/record_encoder/base_test_record_encoder.py b/tests/unit/record_encoder/base_test_record_encoder.py index 7b0ab6fb..0c1c2008 100644 --- a/tests/unit/record_encoder/base_test_record_encoder.py +++ b/tests/unit/record_encoder/base_test_record_encoder.py @@ -2,7 +2,7 @@ import math from abc import ABC, abstractmethod -from resin.knoweldge_base.models import KBDocChunk, KBEncodedDocChunk, KBQuery +from resin.knowledge_base.models import KBDocChunk, KBEncodedDocChunk, KBQuery from resin.models.data_models import Query diff --git a/tests/unit/record_encoder/test_dense_record_encoder.py b/tests/unit/record_encoder/test_dense_record_encoder.py index a5eb2eb4..c2b7cd8a 100644 --- a/tests/unit/record_encoder/test_dense_record_encoder.py +++ b/tests/unit/record_encoder/test_dense_record_encoder.py @@ -1,6 +1,6 @@ import pytest -from resin.knoweldge_base.record_encoder import DenseRecordEncoder +from resin.knowledge_base.record_encoder import DenseRecordEncoder from .base_test_record_encoder import BaseTestRecordEncoder from ..stubs.stub_dense_encoder import StubDenseEncoder diff --git a/tests/unit/record_encoder/test_openai_record_encoder.py b/tests/unit/record_encoder/test_openai_record_encoder.py index 8f213b81..fd5333ec 100644 --- a/tests/unit/record_encoder/test_openai_record_encoder.py +++ b/tests/unit/record_encoder/test_openai_record_encoder.py @@ -1,7 +1,7 @@ import pytest from pinecone_text.dense.openai_encoder import OpenAIEncoder -from resin.knoweldge_base.record_encoder.openai import OpenAIRecordEncoder +from resin.knowledge_base.record_encoder.openai import OpenAIRecordEncoder from .base_test_record_encoder import BaseTestRecordEncoder from unittest.mock import Mock diff --git a/tests/unit/stubs/stub_chunker.py b/tests/unit/stubs/stub_chunker.py index 6f0dd331..aceb214f 100644 --- a/tests/unit/stubs/stub_chunker.py +++ b/tests/unit/stubs/stub_chunker.py @@ -1,6 +1,6 @@ from typing import List -from resin.knoweldge_base.chunker.base import Chunker -from resin.knoweldge_base.models import KBDocChunk +from resin.knowledge_base.chunker.base import Chunker +from resin.knowledge_base.models import KBDocChunk from resin.models.data_models import Document diff --git a/tests/unit/stubs/stub_record_encoder.py b/tests/unit/stubs/stub_record_encoder.py index 3bd49454..9af5d387 100644 --- a/tests/unit/stubs/stub_record_encoder.py +++ b/tests/unit/stubs/stub_record_encoder.py @@ -1,7 +1,7 @@ from typing import List -from resin.knoweldge_base.record_encoder import RecordEncoder -from resin.knoweldge_base.models import KBQuery, KBDocChunk, KBEncodedDocChunk +from resin.knowledge_base.record_encoder import RecordEncoder +from resin.knowledge_base.models import KBQuery, KBDocChunk, KBEncodedDocChunk from resin.models.data_models import Query from .stub_dense_encoder import StubDenseEncoder From 5485001170944d53c7c2153c0af74590daf631c4 Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Thu, 19 Oct 2023 14:24:16 +0300 Subject: [PATCH 23/64] remove unused dependencies --- pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 529c3e87..4553fb17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,9 +17,7 @@ tiktoken = "^0.3.3" pinecone-datasets = "^0.6.1" pydantic = "^1.10.7" pinecone-text = { version = "^0.6.0", extras = ["openai"] } -flake8-pyproject = "^1.2.3" pandas-stubs = "^2.0.3.230814" -langchain = "^0.0.188" fastapi = "^0.92.0" uvicorn = "^0.20.0" tenacity = "^8.2.1" From a971d532acd2df03eb64fadb631ba14626e6b5c9 Mon Sep 17 00:00:00 2001 From: ilai Date: Thu, 19 Oct 2023 15:21:35 +0300 Subject: [PATCH 24/64] [cli] Improve chat errors and help messages --- src/resin_cli/cli.py | 61 ++++++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/src/resin_cli/cli.py b/src/resin_cli/cli.py index f585d476..f7ebd565 100644 --- a/src/resin_cli/cli.py +++ b/src/resin_cli/cli.py @@ -11,6 +11,7 @@ import pandas as pd import openai +from openai.error import APIError as OpenAI_APIError from resin.knoweldge_base import KnowledgeBase from resin.models.data_models import Document @@ -245,11 +246,10 @@ def _chat( openai_response = openai.ChatCompletion.create( model=model, messages=history, stream=stream, api_base=api_base ) - except Exception as e: - msg = "Oops... something went wrong with the LLM" + " the error I got is: " - click.echo(click.style(msg, fg="red"), err=True, nl=False) - click.echo(f"{e}") - sys.exit(1) + except (Exception, OpenAI_APIError) as e: + err = e.http_body if isinstance(e, OpenAI_APIError) else str(e) + msg = f"Oops... something went wrong. The error I got is: {err}" + raise CLIError(msg) end = time.time() duration_in_sec = end - start click.echo(click.style(f"\n> AI {speaker}:\n", fg=speaker_color)) @@ -289,39 +289,44 @@ def _chat( @cli.command( help=( - "Chat allows you to chat with your Resin index, " - + "Chat is a debugging tool, it is not meant to be used for production" + """ + Debugging tool for chatting with the Resin RAG service. + + Run an interactive chat with the Resin RAG service, for debugging and demo + purposes. A prompt is provided for the user to enter a message, and the + RAG-infused ChatBot will respond. You can continue the conversation by entering + more messages. Hit Ctrl+C to exit. + + To compare RAG-infused ChatBot with the original LLM, run with the `--compare` + flag, which would display both models' responses side by side. + """ + ) ) -@click.option("--stream/--no-stream", default=True, help="Stream") -@click.option("--debug/--no-debug", default=False, help="Print debug info") -@click.option( - "--rag/--no-rag", - default=True, - help="Direct chat with the model", -) -@click.option("--chat-service-url", default="http://0.0.0.0:8000") -@click.option( - "--index-name", - default=os.environ.get("INDEX_NAME"), - help="Index name suffix", -) -def chat(index_name, chat_service_url, rag, debug, stream): +@click.option("--stream/--no-stream", default=True, + help="Stream the response from the RAG chatbot word by word") +@click.option("--debug/--no-debug", default=False, + help="Print additional debugging information") +@click.option("--compare/--no-compare", default=False, + help="Compare RAG-infused Chatbot with native LLM",) +@click.option("--chat-service-url", default="http://0.0.0.0:8000", + help="URL of the Resin service to use. Defaults to http://0.0.0.0:8000") +def chat(chat_service_url, compare, debug, stream): check_service_health(chat_service_url) note_msg = ( "🚨 Note 🚨\n" - + "Chat is a debugging tool, it is not meant to be used for production!" + "Chat is a debugging tool, it is not meant to be used for production!" ) for c in note_msg: click.echo(click.style(c, fg="red"), nl=False) time.sleep(0.01) click.echo() note_white_message = ( - "This method should be used by developers to test the model and the data" - + " in development time, in local environment. " - + "For production use cases, we recommend using the" - + " Resin Service or Resin Library directly \n\n" - + "Let's Chat!" + "This method should be used by developers to test the RAG data and model" + "during development. " + "When you are ready to deploy, run the Resin service as a REST API " + "backend for your chatbot UI. \n\n" + "Let's Chat!" ) for c in note_white_message: click.echo(click.style(c, fg="white"), nl=False) @@ -346,7 +351,7 @@ def chat(index_name, chat_service_url, rag, debug, stream): print_debug_info=debug, ) - if not rag: + if compare: _ = _chat( speaker="Without Context (No RAG)", speaker_color="yellow", From 06a97fc09d9cd91516a6a63d279777eb27cb410a Mon Sep 17 00:00:00 2001 From: ilai Date: Thu, 19 Oct 2023 15:49:08 +0300 Subject: [PATCH 25/64] [cli] Improve the error messages and help of start command --- src/resin_cli/cli.py | 40 ++++++++++++++++------------------------ 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/src/resin_cli/cli.py b/src/resin_cli/cli.py index f7ebd565..5403a84e 100644 --- a/src/resin_cli/cli.py +++ b/src/resin_cli/cli.py @@ -100,6 +100,7 @@ def _initialize_tokenizer(): @click.pass_context def cli(ctx): """ + \b CLI for Pinecone Resin. Actively developed by Pinecone. To use the CLI, you need to have a Pinecone account. Visit https://www.pinecone.io/ to sign up for free. @@ -375,32 +376,23 @@ def chat(chat_service_url, compare, debug, stream): @cli.command( help=( - "Start the Resin service, this will start a uvicorn server" - + " that will serve the Resin API." - + " If you are are locally debugging, you can use the --debug flag" - + " to start a new terminal window with the right env vars for `resin chat`" + """ + \b + Start the Resin service. + This command will launch a uvicorn server that will serve the Resin API. + + If you like to try out the chatbot, run `resin chat` in a separate terminal + window. + """ ) ) -@click.option( - "--chat", is_flag=True, help="open a new terminal window for debugging" -) -@click.option("--host", default="0.0.0.0", help="Host") -@click.option("--port", default=8000, help="Port") -@click.option("--ssl/--no-ssl", default=False, help="SSL") -@click.option("--reload/--no-reload", default=False, help="Reload") -def start(chat, host, port, ssl, reload): - if chat: - command_to_run = "clear && echo Welcome to Pinecone Canopy," - + " run *resin chat* to start chatting with your index" - - script = f''' - tell application "Terminal" - activate - do script "{command_to_run}" - end tell - ''' - - subprocess.run(["osascript", "-e", script], env=os.environ.copy()) +@click.option("--host", default="0.0.0.0", + help="Hostname or ip address to bind the server to. Defaults to 0.0.0.0") +@click.option("--port", default=8000, + help="TCP port to bind the server to. Defaults to 8000") +@click.option("--reload/--no-reload", default=False, + help="Set the server to reload on code changes. Defaults to False") +def start(host, port, reload): click.echo(f"Starting Resin service on {host}:{port}") start_service(host, port, reload) From 6fa8e9cb5857fbd605956ea12af9ad1ec3d9e247 Mon Sep 17 00:00:00 2001 From: ilai Date: Thu, 19 Oct 2023 17:35:04 +0300 Subject: [PATCH 26/64] [cli] resin health - take url instead of separate host port Makes much more sense --- src/resin_cli/cli.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/resin_cli/cli.py b/src/resin_cli/cli.py index 5403a84e..b8934ddd 100644 --- a/src/resin_cli/cli.py +++ b/src/resin_cli/cli.py @@ -40,8 +40,7 @@ def check_service_health(url: str): try: - health_url = os.path.join(url, "health") - res = requests.get(health_url) + res = requests.get(url + "/health") res.raise_for_status() return res.ok except requests.exceptions.ConnectionError: @@ -111,14 +110,10 @@ def cli(ctx): @cli.command(help="Check if resin service is running and healthy.") -@click.option("--host", default="0.0.0.0", help="Resin's service hostname") -@click.option("--port", default=8000, help="The port of the resin service") -@click.option("--ssl/--no-ssl", default=False, help="Whether to use ssl for the " - "connection to resin service") -def health(host, port, ssl): - ssl_str = "s" if ssl else "" - service_url = f"http{ssl_str}://{host}:{port}" - check_service_health(service_url) +@click.option("--url", default="http://0.0.0.0:8000", + help="Resin's service url. Defaults to http://0.0.0.0:8000") +def health(url): + check_service_health(url) click.echo(click.style("Resin service is healthy!", fg="green")) return From 83289111dfd2113594846288b50a717219d413b5 Mon Sep 17 00:00:00 2001 From: ilai Date: Thu, 19 Oct 2023 18:08:04 +0300 Subject: [PATCH 27/64] [cli] Change `resin stop` mechanism Instead of finding the PID and killing it, ask the server to kill itself. The server would send a kill signal to its parent process. --- src/resin_cli/app.py | 24 +++++++++++--- src/resin_cli/cli.py | 76 ++++++++++++-------------------------------- 2 files changed, 40 insertions(+), 60 deletions(-) diff --git a/src/resin_cli/app.py b/src/resin_cli/app.py index ab1c743f..9e8c64eb 100644 --- a/src/resin_cli/app.py +++ b/src/resin_cli/app.py @@ -1,7 +1,11 @@ import os import logging +import signal import sys import uuid + +import openai +from multiprocessing import current_process from dotenv import load_dotenv from resin.llm import BaseLLM @@ -23,9 +27,10 @@ ChatRequest, ContextQueryRequest, \ ContextUpsertRequest, HealthStatus, ContextDeleteRequest -load_dotenv() # load env vars before import of openai -from resin.llm.openai import OpenAILLM # noqa: E402 +from resin.llm.openai import OpenAILLM +load_dotenv() # load env vars before import of openai +openai.api_key = os.getenv("OPENAI_API_KEY") app = FastAPI() @@ -157,6 +162,17 @@ async def health_check(): return HealthStatus(pinecone_status="OK", llm_status="OK") +@app.get( + "/shutdown" +) +async def shutdown(): + logger.info("Shutting down") + proc = current_process() + pid = proc._parent_pid if "SpawnProcess" in proc.name else proc.pid + os.kill(pid, signal.SIGINT) + return {"message": "Shutting down"} + + @app.on_event("startup") async def startup(): _init_logging() @@ -197,9 +213,9 @@ def _init_engines(): kb.connect() -def start(host="0.0.0.0", port=8000, reload=False): +def start(host="0.0.0.0", port=8000, reload=False, workers=1): uvicorn.run("resin_cli.app:app", - host=host, port=port, reload=reload) + host=host, port=port, reload=reload, workers=workers) if __name__ == "__main__": diff --git a/src/resin_cli/cli.py b/src/resin_cli/cli.py index b8934ddd..4030ab68 100644 --- a/src/resin_cli/cli.py +++ b/src/resin_cli/cli.py @@ -387,69 +387,33 @@ def chat(chat_service_url, compare, debug, stream): help="TCP port to bind the server to. Defaults to 8000") @click.option("--reload/--no-reload", default=False, help="Set the server to reload on code changes. Defaults to False") -def start(host, port, reload): +@click.option("--workers", default=1, help="Number of worker processes. Defaults to 1") +def start(host, port, reload, workers): click.echo(f"Starting Resin service on {host}:{port}") - start_service(host, port, reload) + start_service(host, port=port, reload=reload, workers=workers) @cli.command( help=( - "Stop the Resin service, this will kill the uvicorn server" - + " that is serving the Resin API." - + " This method is not recommended," - + " as it will kill the server by looking for the PID" - + " of the server, instead, we recommend using" - + " ctrl+c on the terminal where you started" + """ + \b + Stop the Resin service. + This command will send a shutdown request to the Resin service. + """ ) ) -@click.option("--host", default="0.0.0.0", help="Host") -@click.option("--port", default=8000, help="Port") -@click.option("--ssl/--no-ssl", default=False, help="SSL") -def stop(host, port, ssl): - ssl_str = "s" if ssl else "" - service_url = f"http{ssl_str}://{host}:{port}" - - check_service_health(service_url) - - import subprocess - - p1 = subprocess.Popen(["lsof", "-t", "-i", f"tcp:{port}"], stdout=subprocess.PIPE) - running_server_id = p1.stdout.read().decode("utf-8").strip() - if running_server_id == "": - click.echo( - click.style( - "Did not find active process for Resin service" + f" on {host}:{port}", - fg="red", - ) - ) - sys.exit(1) - - msg = ( - "Warning, this will invoke in process kill" - + " to the PID of the service, this method is not recommended!" - + " We recommend ctrl+c on the terminal where you started the service" - + " as this will allow the service to gracefully shutdown" - ) - click.echo(click.style(msg, fg="yellow")) - - click.confirm( - click.style( - f"Stopping Resin service on {host}:{port} with pid " f"{running_server_id}", - fg="red", - ), - abort=True, - ) - p2 = subprocess.Popen( - ["kill", "-9", running_server_id], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - kill_result = p2.stderr.read().decode("utf-8").strip() - if kill_result == "": - click.echo(click.style("Success!", fg="green")) - else: - click.echo(click.style(kill_result, fg="red")) - click.echo(click.style("Failed!", fg="red")) +@click.option("url", "--url", default="http://0.0.0.0:8000", + help="URL of the Resin service to use. Defaults to http://0.0.0.0:8000") +def stop(url): + try: + res = requests.get(url + "/shutdown") + res.raise_for_status() + return res.ok + except requests.exceptions.ConnectionError: + msg = f""" + Could not find Resin service on {url}. + """ + raise CLIError(msg) if __name__ == "__main__": From 54233cc4fed7231204ae1329422171f645445ab3 Mon Sep 17 00:00:00 2001 From: ilai Date: Thu, 19 Oct 2023 18:19:24 +0300 Subject: [PATCH 28/64] [cli] Improve help message + linter Slightly improved the help message --- src/resin_cli/cli.py | 53 ++++++++++++------------ src/resin_cli/data_loader/data_loader.py | 5 ++- tests/unit/cli/test_data_loader.py | 3 +- 3 files changed, 32 insertions(+), 29 deletions(-) diff --git a/src/resin_cli/cli.py b/src/resin_cli/cli.py index 4030ab68..8d7bfb19 100644 --- a/src/resin_cli/cli.py +++ b/src/resin_cli/cli.py @@ -2,8 +2,6 @@ import click import time -import sys -import subprocess import requests from dotenv import load_dotenv @@ -15,7 +13,7 @@ from resin.knoweldge_base import KnowledgeBase from resin.models.data_models import Document -from resin.tokenizer import OpenAITokenizer, Tokenizer +from resin.tokenizer import Tokenizer from resin_cli.data_loader import ( load_from_path, CLIError, @@ -45,7 +43,7 @@ def check_service_health(url: str): return res.ok except requests.exceptions.ConnectionError: msg = f""" - Resin service is not running on {url}. + Resin service is not running on {url}. please run `resin start` """ raise CLIError(msg) @@ -57,6 +55,7 @@ def check_service_health(url: str): ) raise CLIError(msg) + @retry(wait=wait_fixed(5), stop=stop_after_attempt(6)) def wait_for_service(chat_service_url: str): check_service_health(chat_service_url) @@ -121,14 +120,14 @@ def health(url): @cli.command( help=( """Create a new Pinecone index that that will be used by Resin. - - A Resin service can not be started without a Pinecone index which is configured - to work with Resin. - This command will create a new Pinecone index and configure it in the right - schema for Resin. If the embedding's dimension is not explicitly configured by - the config file - the embedding model will be tapped with a single token to - infer the dimensionality of the embedding space. - """ + \b + A Resin service can not be started without a Pinecone index which is configured to work with Resin. + This command will create a new Pinecone index and configure it in the right schema. + + If the embedding vectors' dimension is not explicitly configured in + the config file - the embedding model will be tapped with a single token to + infer the dimensionality of the embedding space. + """ # noqa: E501 ) ) @click.argument("index-name", nargs=1, envvar="INDEX_NAME", type=str, required=True) @@ -151,11 +150,13 @@ def new(index_name): @cli.command( help=( - """Upload local data files containing documents to the Resin service. - - Load all the documents from data file or a directory containing multiple data - files. The allowed formats are .jsonl and .parquet. """ + \b + Upload local data files containing documents to the Resin service. + + Load all the documents from data file or a directory containing multiple data files. + The allowed formats are .jsonl and .parquet. + """ # noqa: E501 ) ) @click.argument("data-path", type=click.Path(exists=True)) @@ -287,12 +288,12 @@ def _chat( help=( """ Debugging tool for chatting with the Resin RAG service. - - Run an interactive chat with the Resin RAG service, for debugging and demo - purposes. A prompt is provided for the user to enter a message, and the + + Run an interactive chat with the Resin RAG service, for debugging and demo + purposes. A prompt is provided for the user to enter a message, and the RAG-infused ChatBot will respond. You can continue the conversation by entering more messages. Hit Ctrl+C to exit. - + To compare RAG-infused ChatBot with the original LLM, run with the `--compare` flag, which would display both models' responses side by side. """ @@ -373,11 +374,11 @@ def chat(chat_service_url, compare, debug, stream): help=( """ \b - Start the Resin service. + Start the Resin service. This command will launch a uvicorn server that will serve the Resin API. - - If you like to try out the chatbot, run `resin chat` in a separate terminal - window. + + If you like to try out the chatbot, run `resin chat` in a separate terminal + window. """ ) ) @@ -397,7 +398,7 @@ def start(host, port, reload, workers): help=( """ \b - Stop the Resin service. + Stop the Resin service. This command will send a shutdown request to the Resin service. """ ) @@ -411,7 +412,7 @@ def stop(url): return res.ok except requests.exceptions.ConnectionError: msg = f""" - Could not find Resin service on {url}. + Could not find Resin service on {url}. """ raise CLIError(msg) diff --git a/src/resin_cli/data_loader/data_loader.py b/src/resin_cli/data_loader/data_loader.py index 7ffcaade..2483e33f 100644 --- a/src/resin_cli/data_loader/data_loader.py +++ b/src/resin_cli/data_loader/data_loader.py @@ -20,13 +20,16 @@ class IDsNotUniqueError(ValueError): class DocumentsValidationError(ValueError): pass -format_multiline = lambda s: dedent(s).strip() + +def format_multiline(msg): + return dedent(msg).strip() class CLIError(ClickException): def format_message(self) -> str: return click.style(format_multiline(self.message), fg='red') + def _process_metadata(value): if pd.isna(value): return {} diff --git a/tests/unit/cli/test_data_loader.py b/tests/unit/cli/test_data_loader.py index a7b08f09..eb6bd511 100644 --- a/tests/unit/cli/test_data_loader.py +++ b/tests/unit/cli/test_data_loader.py @@ -133,7 +133,6 @@ ) - bad_df_has_excess_field = ( pd.DataFrame( [ @@ -171,7 +170,7 @@ {"id": 3, "text": "baz", "sorce": "baz_source"}, ] ), - DocumentsValidationError + DocumentsValidationError ) bad_df_missing_mandatory_field = ( From 6f1b987e31adf1ed3b87ae16c4310138b626fe79 Mon Sep 17 00:00:00 2001 From: ilai Date: Sun, 22 Oct 2023 10:25:18 +0300 Subject: [PATCH 29/64] [cli] Fix PR comments - Use urljoin - Rename `--compare` to `--baseline` - Improve some error messages --- src/resin_cli/cli.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/resin_cli/cli.py b/src/resin_cli/cli.py index 2e433aa1..b1635a9a 100644 --- a/src/resin_cli/cli.py +++ b/src/resin_cli/cli.py @@ -10,6 +10,7 @@ import pandas as pd import openai from openai.error import APIError as OpenAI_APIError +from urllib.parse import urljoin from resin.knoweldge_base import KnowledgeBase from resin.models.data_models import Document @@ -38,7 +39,7 @@ def check_service_health(url: str): try: - res = requests.get(url + "/health") + res = requests.get(urljoin(url, "/health")) res.raise_for_status() return res.ok except requests.exceptions.ConnectionError: @@ -168,9 +169,10 @@ def new(index_name): ) def upsert(index_name, data_path): if index_name is None: - msg = ("Index name is not provided, please provide it with" + - ' --index-name or set it with env var + ' - '`export INDEX_NAME="MY_INDEX_NAME`') + msg = ( + "No index name provided. Please set --index-name or INDEX_NAME environment " + "variable." + ) raise CLIError(msg) _initialize_tokenizer() @@ -295,7 +297,7 @@ def _chat( RAG-infused ChatBot will respond. You can continue the conversation by entering more messages. Hit Ctrl+C to exit. - To compare RAG-infused ChatBot with the original LLM, run with the `--compare` + To compare RAG-infused ChatBot with the original LLM, run with the `--baseline` flag, which would display both models' responses side by side. """ @@ -305,8 +307,8 @@ def _chat( help="Stream the response from the RAG chatbot word by word") @click.option("--debug/--no-debug", default=False, help="Print additional debugging information") -@click.option("--compare/--no-compare", default=False, - help="Compare RAG-infused Chatbot with native LLM",) +@click.option("--baseline/--no-baseline", default=False, + help="Compare RAG-infused Chatbot with baseline LLM",) @click.option("--chat-service-url", default="http://0.0.0.0:8000", help="URL of the Resin service to use. Defaults to http://0.0.0.0:8000") def chat(chat_service_url, compare, debug, stream): @@ -408,7 +410,7 @@ def start(host, port, reload, workers): help="URL of the Resin service to use. Defaults to http://0.0.0.0:8000") def stop(url): try: - res = requests.get(url + "/shutdown") + res = requests.get(urljoin(url, "/shutdown")) res.raise_for_status() return res.ok except requests.exceptions.ConnectionError: From e9fdc465d34ddfcd437f89d2e5c5b2b5e23f1f2f Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Sun, 22 Oct 2023 11:15:14 +0300 Subject: [PATCH 30/64] set _check_return_type to true --- src/resin/knoweldge_base/knowledge_base.py | 5 +- .../knowledge_base/test_knowledge_base.py | 53 +++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/src/resin/knoweldge_base/knowledge_base.py b/src/resin/knoweldge_base/knowledge_base.py index c0871341..3440d7bd 100644 --- a/src/resin/knoweldge_base/knowledge_base.py +++ b/src/resin/knoweldge_base/knowledge_base.py @@ -432,13 +432,16 @@ def _query_index(self, metadata_filter.update(global_metadata_filter) top_k = query.top_k if query.top_k else self._default_top_k + query_params = deepcopy(query.query_params) + _check_return_type = query.query_params.pop('_check_return_type', False) result = self._index.query(vector=query.values, sparse_vector=query.sparse_values, top_k=top_k, namespace=query.namespace, metadata_filter=metadata_filter, include_metadata=True, - **query.query_params) + _check_return_type=_check_return_type, + **query_params) documents: List[KBDocChunkWithScore] = [] for match in result['matches']: metadata = match['metadata'] diff --git a/tests/system/knowledge_base/test_knowledge_base.py b/tests/system/knowledge_base/test_knowledge_base.py index 3ede466a..a833db6c 100644 --- a/tests/system/knowledge_base/test_knowledge_base.py +++ b/tests/system/knowledge_base/test_knowledge_base.py @@ -158,6 +158,28 @@ def encoded_chunks_large(documents_large, chunker, encoder): return encoder.encode_documents(chunks) +@pytest.fixture +def edge_case_documents(): + return [Document(id="doc_0", + text="", + source="source_1"), + Document(id="doc_1", + text="document with datetime metadata", + source="source_1", + metadata={"datetime": "2021-01-01T00:00:00Z", + "datetime_other_format": "January 1, 2021 00:00:00", + "datetime_other_format_2": "2210.03945"}), + Document(id="2021-01-01T00:00:00Z", + text="id is datetime", + source="source_1")] + + +@pytest.fixture +def edge_case_encoded_chunks(edge_case_documents, chunker, encoder): + chunks = chunker.chunk_documents(edge_case_documents) + return encoder.encode_documents(chunks) + + @pytest.fixture def encoded_chunks(documents, chunker, encoder): chunks = chunker.chunk_documents(documents) @@ -302,6 +324,37 @@ def test_delete_large_df_happy_path(knowledge_base, for chunk in chunks_for_validation]) +def test_upsert_edge_case_documents(knowledge_base, + edge_case_documents, + edge_case_encoded_chunks): + knowledge_base.upsert(edge_case_documents) + + assert_ids_in_index(knowledge_base, [chunk.id + for chunk in edge_case_encoded_chunks]) + + +def test_query_edge_case_documents(knowledge_base, + edge_case_documents, + edge_case_encoded_chunks): + queries = [Query(text=chunk.text, top_k=2) for chunk in edge_case_encoded_chunks] + query_results = knowledge_base.query(queries) + + assert len(query_results) == len(queries) + + for i, q_res in enumerate(query_results): + assert queries[i].text == q_res.query + assert len(q_res.documents) == 2 + q_res.documents[0].score = round(q_res.documents[0].score, 2) + assert q_res.documents[0] == DocumentWithScore( + id=edge_case_encoded_chunks[i].id, + text=edge_case_encoded_chunks[i].text, + metadata=edge_case_encoded_chunks[i].metadata, + source=edge_case_encoded_chunks[i].source, + score=1.0), \ + f"query {i} - expected: {edge_case_encoded_chunks[i]}, " \ + f"actual: {q_res.documents[0]}" + + def test_create_existing_index_no_connect(index_full_name, index_name): kb = KnowledgeBase( index_name=index_name, From f3788f538efdbb6b0887d2aa5407ea68208eabe0 Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Sun, 22 Oct 2023 13:11:35 +0300 Subject: [PATCH 31/64] rename --- .../knowledge_base/test_knowledge_base.py | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/system/knowledge_base/test_knowledge_base.py b/tests/system/knowledge_base/test_knowledge_base.py index a833db6c..c907669e 100644 --- a/tests/system/knowledge_base/test_knowledge_base.py +++ b/tests/system/knowledge_base/test_knowledge_base.py @@ -159,11 +159,8 @@ def encoded_chunks_large(documents_large, chunker, encoder): @pytest.fixture -def edge_case_documents(): - return [Document(id="doc_0", - text="", - source="source_1"), - Document(id="doc_1", +def documents_with_datetime_metadata(): + return [Document(id="doc_1", text="document with datetime metadata", source="source_1", metadata={"datetime": "2021-01-01T00:00:00Z", @@ -175,8 +172,10 @@ def edge_case_documents(): @pytest.fixture -def edge_case_encoded_chunks(edge_case_documents, chunker, encoder): - chunks = chunker.chunk_documents(edge_case_documents) +def datetime_metadata_encoded_chunks(documents_with_datetime_metadata, + chunker, + encoder): + chunks = chunker.chunk_documents(documents_with_datetime_metadata) return encoder.encode_documents(chunks) @@ -324,19 +323,20 @@ def test_delete_large_df_happy_path(knowledge_base, for chunk in chunks_for_validation]) -def test_upsert_edge_case_documents(knowledge_base, - edge_case_documents, - edge_case_encoded_chunks): - knowledge_base.upsert(edge_case_documents) +def test_upsert_documents_with_datetime_metadata(knowledge_base, + documents_with_datetime_metadata, + datetime_metadata_encoded_chunks): + knowledge_base.upsert(documents_with_datetime_metadata) assert_ids_in_index(knowledge_base, [chunk.id - for chunk in edge_case_encoded_chunks]) + for chunk in datetime_metadata_encoded_chunks]) def test_query_edge_case_documents(knowledge_base, - edge_case_documents, - edge_case_encoded_chunks): - queries = [Query(text=chunk.text, top_k=2) for chunk in edge_case_encoded_chunks] + documents_with_datetime_metadata, + datetime_metadata_encoded_chunks): + queries = [Query(text=chunk.text, top_k=2) + for chunk in datetime_metadata_encoded_chunks] query_results = knowledge_base.query(queries) assert len(query_results) == len(queries) @@ -346,12 +346,12 @@ def test_query_edge_case_documents(knowledge_base, assert len(q_res.documents) == 2 q_res.documents[0].score = round(q_res.documents[0].score, 2) assert q_res.documents[0] == DocumentWithScore( - id=edge_case_encoded_chunks[i].id, - text=edge_case_encoded_chunks[i].text, - metadata=edge_case_encoded_chunks[i].metadata, - source=edge_case_encoded_chunks[i].source, + id=datetime_metadata_encoded_chunks[i].id, + text=datetime_metadata_encoded_chunks[i].text, + metadata=datetime_metadata_encoded_chunks[i].metadata, + source=datetime_metadata_encoded_chunks[i].source, score=1.0), \ - f"query {i} - expected: {edge_case_encoded_chunks[i]}, " \ + f"query {i} - expected: {datetime_metadata_encoded_chunks[i]}, " \ f"actual: {q_res.documents[0]}" From b7309098e7b2e7ff376dc18f4708f5d4fd1d535a Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Sun, 22 Oct 2023 13:42:10 +0300 Subject: [PATCH 32/64] fix kb tests --- tests/system/knowledge_base/test_knowledge_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system/knowledge_base/test_knowledge_base.py b/tests/system/knowledge_base/test_knowledge_base.py index 42589c4d..8cd8a7e7 100644 --- a/tests/system/knowledge_base/test_knowledge_base.py +++ b/tests/system/knowledge_base/test_knowledge_base.py @@ -160,7 +160,7 @@ def encoded_chunks_large(documents_large, chunker, encoder): @pytest.fixture def documents_with_datetime_metadata(): - return [Document(id="doc_1", + return [Document(id="doc_1_metadata", text="document with datetime metadata", source="source_1", metadata={"datetime": "2021-01-01T00:00:00Z", @@ -352,7 +352,7 @@ def test_query_edge_case_documents(knowledge_base, source=datetime_metadata_encoded_chunks[i].source, score=1.0), \ f"query {i} - expected: {datetime_metadata_encoded_chunks[i]}, " \ - f"actual: {q_res.documents[0]}" + f"actual: {q_res.documents}" def test_create_existing_index_no_connect(index_full_name, index_name): From 9dec071c32bb1d0a56c1c653d71dcd505ad8908d Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Sun, 22 Oct 2023 14:15:40 +0300 Subject: [PATCH 33/64] add retry to query tests --- .../knowledge_base/test_knowledge_base.py | 65 +++++++------------ 1 file changed, 24 insertions(+), 41 deletions(-) diff --git a/tests/system/knowledge_base/test_knowledge_base.py b/tests/system/knowledge_base/test_knowledge_base.py index 8cd8a7e7..e157edf4 100644 --- a/tests/system/knowledge_base/test_knowledge_base.py +++ b/tests/system/knowledge_base/test_knowledge_base.py @@ -117,6 +117,28 @@ def assert_ids_not_in_index(knowledge_base, ids): assert len(fetch_result) == 0, f"Found unexpected ids: {len(fetch_result.keys())}" +@retry_decorator() +def execute_and_assert_queries(knowledge_base, chunks_to_query): + queries = [Query(text=chunk.text, top_k=2) for chunk in chunks_to_query] + + query_results = knowledge_base.query(queries) + + assert len(query_results) == len(queries) + + for i, q_res in enumerate(query_results): + assert queries[i].text == q_res.query + assert len(q_res.documents) == 2 + q_res.documents[0].score = round(q_res.documents[0].score, 2) + assert q_res.documents[0] == DocumentWithScore( + id=chunks_to_query[i].id, + text=chunks_to_query[i].text, + metadata=chunks_to_query[i].metadata, + source=chunks_to_query[i].source, + score=1.0), \ + f"query {i} - expected: {chunks_to_query[i]}, " \ + f"actual: {q_res.documents}" + + @pytest.fixture(scope="module", autouse=True) def teardown_knowledge_base(index_full_name, knowledge_base): yield @@ -224,28 +246,7 @@ def test_upsert_forbidden_metadata(knowledge_base, documents, key): def test_query(knowledge_base, encoded_chunks): - queries = [Query(text=encoded_chunks[0].text), - Query(text=encoded_chunks[1].text, top_k=2)] - query_results = knowledge_base.query(queries) - - assert len(query_results) == 2 - - expected_top_k = [5, 2] - expected_first_results = [DocumentWithScore(id=chunk.id, - text=chunk.text, - metadata=chunk.metadata, - source=chunk.source, - score=1.0) - for chunk in encoded_chunks[:2]] - for i, q_res in enumerate(query_results): - assert queries[i].text == q_res.query - assert len(q_res.documents) == expected_top_k[i] - q_res.documents[0].score = round(q_res.documents[0].score, 2) - assert q_res.documents[0] == expected_first_results[i] - q_res.documents[0].score = round(q_res.documents[0].score, 2) - assert q_res.documents[0] == expected_first_results[i], \ - f"query {i} - expected: {expected_first_results[i]}, " \ - f"actual: {q_res.documents[0]}" + execute_and_assert_queries(knowledge_base, encoded_chunks) def test_delete_documents(knowledge_base, encoded_chunks): @@ -333,26 +334,8 @@ def test_upsert_documents_with_datetime_metadata(knowledge_base, def test_query_edge_case_documents(knowledge_base, - documents_with_datetime_metadata, datetime_metadata_encoded_chunks): - queries = [Query(text=chunk.text, top_k=2) - for chunk in datetime_metadata_encoded_chunks] - query_results = knowledge_base.query(queries) - - assert len(query_results) == len(queries) - - for i, q_res in enumerate(query_results): - assert queries[i].text == q_res.query - assert len(q_res.documents) == 2 - q_res.documents[0].score = round(q_res.documents[0].score, 2) - assert q_res.documents[0] == DocumentWithScore( - id=datetime_metadata_encoded_chunks[i].id, - text=datetime_metadata_encoded_chunks[i].text, - metadata=datetime_metadata_encoded_chunks[i].metadata, - source=datetime_metadata_encoded_chunks[i].source, - score=1.0), \ - f"query {i} - expected: {datetime_metadata_encoded_chunks[i]}, " \ - f"actual: {q_res.documents}" + execute_and_assert_queries(knowledge_base, datetime_metadata_encoded_chunks) def test_create_existing_index_no_connect(index_full_name, index_name): From 6511175e1b625369f548235410e7ff0fdd5e671d Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Sun, 22 Oct 2023 16:51:41 +0300 Subject: [PATCH 34/64] fix test --- tests/system/knowledge_base/test_knowledge_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system/knowledge_base/test_knowledge_base.py b/tests/system/knowledge_base/test_knowledge_base.py index e157edf4..d4bfd74e 100644 --- a/tests/system/knowledge_base/test_knowledge_base.py +++ b/tests/system/knowledge_base/test_knowledge_base.py @@ -128,7 +128,7 @@ def execute_and_assert_queries(knowledge_base, chunks_to_query): for i, q_res in enumerate(query_results): assert queries[i].text == q_res.query assert len(q_res.documents) == 2 - q_res.documents[0].score = round(q_res.documents[0].score, 2) + q_res.documents[0].score = round(q_res.documents[0].score, 1) assert q_res.documents[0] == DocumentWithScore( id=chunks_to_query[i].id, text=chunks_to_query[i].text, From e94f09bf3200382d5183013571ebad7c8f778a11 Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Sun, 22 Oct 2023 17:10:11 +0300 Subject: [PATCH 35/64] add progress bar to upsert from CLI --- src/resin_cli/cli.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/resin_cli/cli.py b/src/resin_cli/cli.py index 04796c6b..a58a8371 100644 --- a/src/resin_cli/cli.py +++ b/src/resin_cli/cli.py @@ -6,6 +6,7 @@ import requests from dotenv import load_dotenv +from tqdm import tqdm import pandas as pd import openai @@ -119,7 +120,8 @@ def new(index_name, tokenizer_model): help="Index name", ) @click.option("--tokenizer-model", default="gpt-3.5-turbo", help="Tokenizer model") -def upsert(index_name, data_path, tokenizer_model): +@click.option("--batch-size", default=10, help="Batch size for upsert") +def upsert(index_name, data_path, tokenizer_model, batch_size): if index_name is None: msg = ("Index name is not provided, please provide it with" + ' --index-name or set it with env var + ' @@ -178,7 +180,13 @@ def upsert(index_name, data_path, tokenizer_model): click.echo(click.style(f"\nTotal records: {len(data)}")) click.confirm(click.style("\nDoes this data look right?", fg="red"), abort=True) - kb.upsert(data) + + pbar = tqdm(total=len(data), desc="Upserting documents") + for i in range(0, len(data), batch_size): + batch = data[i:i + batch_size] + kb.upsert(batch) + pbar.update(len(batch)) + click.echo(click.style("Success!", fg="green")) From 36d778f0ea585943ac15784465a5bc82a8457799 Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Sun, 22 Oct 2023 17:18:08 +0300 Subject: [PATCH 36/64] add tqdm stubs --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 529c3e87..484b74a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ fastapi = "^0.92.0" uvicorn = "^0.20.0" tenacity = "^8.2.1" sse-starlette = "^1.6.5" +types-tqdm = "^4.61.0" [tool.poetry.group.dev.dependencies] From 928202c86eb188eba42d6c3092a5b4c414090080 Mon Sep 17 00:00:00 2001 From: Roy Miara Date: Sun, 22 Oct 2023 23:19:11 +0300 Subject: [PATCH 37/64] fix chat function call --- src/resin_cli/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resin_cli/cli.py b/src/resin_cli/cli.py index b1635a9a..fc3fc6c3 100644 --- a/src/resin_cli/cli.py +++ b/src/resin_cli/cli.py @@ -311,7 +311,7 @@ def _chat( help="Compare RAG-infused Chatbot with baseline LLM",) @click.option("--chat-service-url", default="http://0.0.0.0:8000", help="URL of the Resin service to use. Defaults to http://0.0.0.0:8000") -def chat(chat_service_url, compare, debug, stream): +def chat(chat_service_url, baseline, debug, stream): check_service_health(chat_service_url) note_msg = ( "🚨 Note 🚨\n" From bd0c5d4958b92c9dabdaf67c0c78499bd89691b7 Mon Sep 17 00:00:00 2001 From: ilai Date: Mon, 23 Oct 2023 09:50:29 +0300 Subject: [PATCH 38/64] [cli] Bug fix - forgot to change `compare` to `baseline` --- src/resin_cli/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resin_cli/cli.py b/src/resin_cli/cli.py index fc3fc6c3..3dbbbf82 100644 --- a/src/resin_cli/cli.py +++ b/src/resin_cli/cli.py @@ -351,7 +351,7 @@ def chat(chat_service_url, baseline, debug, stream): print_debug_info=debug, ) - if compare: + if baseline: _ = _chat( speaker="Without Context (No RAG)", speaker_color="yellow", From 907abfdb3ae553115a0dec2dfdef5d29944fcb66 Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Mon, 23 Oct 2023 12:13:09 +0300 Subject: [PATCH 39/64] move index verification to inner connect method --- src/resin/knowledge_base/knowledge_base.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/resin/knowledge_base/knowledge_base.py b/src/resin/knowledge_base/knowledge_base.py index 15b2585e..8ee864d4 100644 --- a/src/resin/knowledge_base/knowledge_base.py +++ b/src/resin/knowledge_base/knowledge_base.py @@ -168,7 +168,7 @@ def _connect_pinecone(): def _connect_index(self, connect_pinecone: bool = True - ) -> Index: + ) -> None: if connect_pinecone: self._connect_pinecone() @@ -186,7 +186,8 @@ def _connect_index(self, f"Unexpected error while connecting to index {self.index_name}. " f"Please check your credentials and try again." ) from e - return index + self.verify_index_connection() + self._index = index @property def _connection_error_msg(self) -> str: @@ -210,8 +211,7 @@ def connect(self) -> None: RuntimeError: If the knowledge base failed to connect to the underlying Pinecone index. """ # noqa: E501 if self._index is None: - self._index = self._connect_index() - self.verify_index_connection() + self._connect_index() def verify_index_connection(self) -> None: """ @@ -320,7 +320,7 @@ def _wait_for_index_provision(self): start_time = time.time() while True: try: - self._index = self._connect_index(connect_pinecone=False) + self._connect_index(connect_pinecone=False) break except RuntimeError: pass From 3bf221b97642f4093a8e59ae0c13aff1b23a4df4 Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Mon, 23 Oct 2023 12:17:42 +0300 Subject: [PATCH 40/64] set index to None if provision failed --- src/resin/knowledge_base/knowledge_base.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/resin/knowledge_base/knowledge_base.py b/src/resin/knowledge_base/knowledge_base.py index 8ee864d4..528e5fb8 100644 --- a/src/resin/knowledge_base/knowledge_base.py +++ b/src/resin/knowledge_base/knowledge_base.py @@ -186,9 +186,14 @@ def _connect_index(self, f"Unexpected error while connecting to index {self.index_name}. " f"Please check your credentials and try again." ) from e - self.verify_index_connection() self._index = index + try: + self.verify_index_connection() + except Exception as e: + self._index = None + raise e + @property def _connection_error_msg(self) -> str: return ( From e6ad1ed889869d5f00cd6305fa287cf81347dfca Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Mon, 23 Oct 2023 12:18:34 +0300 Subject: [PATCH 41/64] shorten code --- src/resin/knowledge_base/knowledge_base.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/resin/knowledge_base/knowledge_base.py b/src/resin/knowledge_base/knowledge_base.py index 528e5fb8..a2cc6a33 100644 --- a/src/resin/knowledge_base/knowledge_base.py +++ b/src/resin/knowledge_base/knowledge_base.py @@ -180,13 +180,12 @@ def _connect_index(self, ) try: - index = Index(index_name=self.index_name) + self._index = Index(index_name=self.index_name) except Exception as e: raise RuntimeError( f"Unexpected error while connecting to index {self.index_name}. " f"Please check your credentials and try again." ) from e - self._index = index try: self.verify_index_connection() From 50e2ce8dc631a2302f66add13f50205031589734 Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Mon, 23 Oct 2023 12:20:36 +0300 Subject: [PATCH 42/64] shorten code --- src/resin/knowledge_base/knowledge_base.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/resin/knowledge_base/knowledge_base.py b/src/resin/knowledge_base/knowledge_base.py index a2cc6a33..bcab064c 100644 --- a/src/resin/knowledge_base/knowledge_base.py +++ b/src/resin/knowledge_base/knowledge_base.py @@ -181,18 +181,14 @@ def _connect_index(self, try: self._index = Index(index_name=self.index_name) + self.verify_index_connection() except Exception as e: + self._index = None raise RuntimeError( f"Unexpected error while connecting to index {self.index_name}. " f"Please check your credentials and try again." ) from e - try: - self.verify_index_connection() - except Exception as e: - self._index = None - raise e - @property def _connection_error_msg(self) -> str: return ( From b666f79a45a8f334430f8ff6b36c463bb8909f9d Mon Sep 17 00:00:00 2001 From: ilai Date: Mon, 23 Oct 2023 12:44:55 +0300 Subject: [PATCH 43/64] [cli] Added option not to stop on the first error during CLI upsert Now that we're upserting in batches anyway, it makes more sense to allow the user to ignore errors and continue to upsert other documents --- src/resin_cli/cli.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/resin_cli/cli.py b/src/resin_cli/cli.py index e5bda7f7..14f9aa6a 100644 --- a/src/resin_cli/cli.py +++ b/src/resin_cli/cli.py @@ -168,8 +168,11 @@ def new(index_name): help="The name of the index to upload the data to. " "Inferred from INDEX_NAME env var if not provided." ) -@click.option("--batch-size", default=10, help="Batch size for upsert") -def upsert(index_name, data_path, batch_size): +@click.option("--batch-size", default=10, + help="Number of documents to upload in each batch. Defaults to 10.") +@click.option("--stop-on-error/--no-stop-on-error", default=True, + help="Whether to stop when the first error arises. Defaults to True.") +def upsert(index_name: str, data_path: str, batch_size: int, stop_on_error: bool): if index_name is None: msg = ( "No index name provided. Please set --index-name or INDEX_NAME environment " @@ -224,18 +227,30 @@ def upsert(index_name, data_path, batch_size): abort=True) pbar = tqdm(total=len(data), desc="Upserting documents") + failed_docs = [] for i in range(0, len(data), batch_size): batch = data[i:i + batch_size] try: kb.upsert(data) except Exception as e: - msg = ( - f"Failed to upsert data to index {kb.index_name}. Underlying error: {e}" - ) - raise CLIError(msg) + if stop_on_error: + msg = ( + f"Failed to upsert data to index {kb.index_name}. " + f"Underlying error: {e}" + ) + raise CLIError(msg) + else: + failed_docs.extend([_.id for _ in batch]) pbar.update(len(batch)) + if failed_docs: + msg = ( + f"Failed to upsert the following documents to index {kb.index_name}: " + f"{failed_docs}" + ) + raise CLIError(msg) + click.echo(click.style("Success!", fg="green")) From 928373fd2e6d6663f4f0c19ecac19eec49dd7d44 Mon Sep 17 00:00:00 2001 From: ilai Date: Mon, 23 Oct 2023 12:49:19 +0300 Subject: [PATCH 44/64] [cli] Upsert 'stop-on-error' - added tolerance of 10% of the data If more than 10% of the data has failed - stop trying to upsert more documents. --- src/resin_cli/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resin_cli/cli.py b/src/resin_cli/cli.py index 14f9aa6a..a29b48c0 100644 --- a/src/resin_cli/cli.py +++ b/src/resin_cli/cli.py @@ -233,7 +233,7 @@ def upsert(index_name: str, data_path: str, batch_size: int, stop_on_error: bool try: kb.upsert(data) except Exception as e: - if stop_on_error: + if stop_on_error or len(failed_docs) > len(data) // 10: msg = ( f"Failed to upsert data to index {kb.index_name}. " f"Underlying error: {e}" From bdccd2e053758fa938cd82b003232fba0444d21c Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Mon, 23 Oct 2023 15:36:50 +0300 Subject: [PATCH 45/64] initial workflow --- .github/workflows/pre-release-CI.yml | 47 ++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/pre-release-CI.yml diff --git a/.github/workflows/pre-release-CI.yml b/.github/workflows/pre-release-CI.yml new file mode 100644 index 00000000..e33bd2c1 --- /dev/null +++ b/.github/workflows/pre-release-CI.yml @@ -0,0 +1,47 @@ +name: Pre Release CI + +on: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + build-and-test: + name: Build & Test on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: 3.9 + + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + version: 1.3.2 + + - name: Build wheel + run: | + poetry build + + - name: Install the wheel + run: | + pip install dist/*.whl + + - name: Run tests + run: pytest --html=report.html --self-contained-html tests/unit + + - name: Upload pytest reports + if: always() + uses: actions/upload-artifact@v3 + with: + name: pytest-report-${{ matrix.os }}-py${{ matrix.python-version }} + path: .pytest_cache From a84c90edd5376cd8931ced5f757eea286ef108f4 Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Mon, 23 Oct 2023 15:42:28 +0300 Subject: [PATCH 46/64] run workflow on current branch for testing --- .github/workflows/pre-release-CI.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pre-release-CI.yml b/.github/workflows/pre-release-CI.yml index e33bd2c1..e74b2889 100644 --- a/.github/workflows/pre-release-CI.yml +++ b/.github/workflows/pre-release-CI.yml @@ -1,7 +1,9 @@ name: Pre Release CI on: - workflow_dispatch: + push: + branches: + - "pre-release-CI" concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} From 824e729e4cb21deb4a18af8a4d13d25b16a4ca46 Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Mon, 23 Oct 2023 15:49:54 +0300 Subject: [PATCH 47/64] install pytest --- .github/workflows/pre-release-CI.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pre-release-CI.yml b/.github/workflows/pre-release-CI.yml index e74b2889..c53d97e3 100644 --- a/.github/workflows/pre-release-CI.yml +++ b/.github/workflows/pre-release-CI.yml @@ -38,6 +38,10 @@ jobs: run: | pip install dist/*.whl + - name: Install pytest + run: | + poetry install pytest + - name: Run tests run: pytest --html=report.html --self-contained-html tests/unit From 064de2f272358a579eb29f3457e4208d1b9d5a4c Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Mon, 23 Oct 2023 15:55:01 +0300 Subject: [PATCH 48/64] Install tests dependencies --- .github/workflows/pre-release-CI.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pre-release-CI.yml b/.github/workflows/pre-release-CI.yml index c53d97e3..6b69b2a4 100644 --- a/.github/workflows/pre-release-CI.yml +++ b/.github/workflows/pre-release-CI.yml @@ -38,9 +38,9 @@ jobs: run: | pip install dist/*.whl - - name: Install pytest + - name: Install tests dependencies run: | - poetry install pytest + pip install pytest pytest-html asyncio pytest-asyncio pytest-xdist - name: Run tests run: pytest --html=report.html --self-contained-html tests/unit From 891d90a7f495a9f282cf57b1bc56c9c4c34088da Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Mon, 23 Oct 2023 15:58:52 +0300 Subject: [PATCH 49/64] temp disable in progress cancelling --- .github/workflows/pre-release-CI.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pre-release-CI.yml b/.github/workflows/pre-release-CI.yml index 6b69b2a4..25adf702 100644 --- a/.github/workflows/pre-release-CI.yml +++ b/.github/workflows/pre-release-CI.yml @@ -7,7 +7,7 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true + cancel-in-progress: false jobs: build-and-test: @@ -40,7 +40,7 @@ jobs: - name: Install tests dependencies run: | - pip install pytest pytest-html asyncio pytest-asyncio pytest-xdist + pip install pytest pytest-html asyncio pytest-asyncio pytest-xdist pytest-mock - name: Run tests run: pytest --html=report.html --self-contained-html tests/unit From 39722d175c53babffc16caa1321aa3e355428619 Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Mon, 23 Oct 2023 16:02:18 +0300 Subject: [PATCH 50/64] temp remove windows --- .github/workflows/pre-release-CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-release-CI.yml b/.github/workflows/pre-release-CI.yml index 25adf702..79207ea1 100644 --- a/.github/workflows/pre-release-CI.yml +++ b/.github/workflows/pre-release-CI.yml @@ -15,7 +15,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest, macos-latest] steps: - uses: actions/checkout@v3 From 6eca830622d80d27529181388e7901152935a8d9 Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Mon, 23 Oct 2023 16:06:31 +0300 Subject: [PATCH 51/64] fix windows --- .github/workflows/pre-release-CI.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pre-release-CI.yml b/.github/workflows/pre-release-CI.yml index 79207ea1..c0b48598 100644 --- a/.github/workflows/pre-release-CI.yml +++ b/.github/workflows/pre-release-CI.yml @@ -15,7 +15,10 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, windows-latest, macos-latest] + defaults: + run: + shell: bash steps: - uses: actions/checkout@v3 From fe4ff0084234a9aaaa52e7d9d420ebf67b357f31 Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Mon, 23 Oct 2023 16:20:16 +0300 Subject: [PATCH 52/64] revert dispatch --- .github/workflows/pre-release-CI.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/pre-release-CI.yml b/.github/workflows/pre-release-CI.yml index c0b48598..77faedbb 100644 --- a/.github/workflows/pre-release-CI.yml +++ b/.github/workflows/pre-release-CI.yml @@ -1,9 +1,7 @@ name: Pre Release CI on: - push: - branches: - - "pre-release-CI" + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} From 92ae5d2daaa8cdff8a39e0811018767ecf4f1f3d Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Mon, 23 Oct 2023 16:21:38 +0300 Subject: [PATCH 53/64] enable cancel in progress --- .github/workflows/pre-release-CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-release-CI.yml b/.github/workflows/pre-release-CI.yml index 77faedbb..7ef338da 100644 --- a/.github/workflows/pre-release-CI.yml +++ b/.github/workflows/pre-release-CI.yml @@ -5,7 +5,7 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: false + cancel-in-progress: true jobs: build-and-test: From a886785c71a37e7592bb8b617f86d5e87c836096 Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Mon, 23 Oct 2023 16:45:08 +0300 Subject: [PATCH 54/64] fix mypy issue --- src/resin_cli/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resin_cli/cli.py b/src/resin_cli/cli.py index bbe77ea7..e8d4bc5a 100644 --- a/src/resin_cli/cli.py +++ b/src/resin_cli/cli.py @@ -51,7 +51,7 @@ def check_service_health(url: str): raise CLIError(msg) except requests.exceptions.HTTPError as e: - error = e.response.json().get("detail", None) or e.response.text + error = e.response.json().get("detail", "") or e.response.text msg = ( f"Resin service on {url} is not healthy, failed with error: {error}" ) From 40a3efdb325c5debac4a5ccb0468fc647efca3c6 Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Mon, 23 Oct 2023 16:54:59 +0300 Subject: [PATCH 55/64] resolve mypy --- src/resin_cli/cli.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/resin_cli/cli.py b/src/resin_cli/cli.py index e8d4bc5a..aba635b7 100644 --- a/src/resin_cli/cli.py +++ b/src/resin_cli/cli.py @@ -51,7 +51,10 @@ def check_service_health(url: str): raise CLIError(msg) except requests.exceptions.HTTPError as e: - error = e.response.json().get("detail", "") or e.response.text + if e.response is not None: + error = e.response.json().get("detail", "") or e.response.text + else: + error = str(e) msg = ( f"Resin service on {url} is not healthy, failed with error: {error}" ) From ffa46664b658b403dc077f594405d07c062b7cb6 Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Mon, 23 Oct 2023 18:20:09 +0300 Subject: [PATCH 56/64] run on several python versions --- .github/workflows/pre-release-CI.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pre-release-CI.yml b/.github/workflows/pre-release-CI.yml index 7ef338da..d272b94a 100644 --- a/.github/workflows/pre-release-CI.yml +++ b/.github/workflows/pre-release-CI.yml @@ -14,6 +14,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macos-latest] + python-version: [3.9, '3.10', 3.11] defaults: run: shell: bash @@ -24,7 +25,7 @@ jobs: - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: ${{ matrix.python-version }} - name: Install Poetry uses: snok/install-poetry@v1 @@ -52,3 +53,4 @@ jobs: with: name: pytest-report-${{ matrix.os }}-py${{ matrix.python-version }} path: .pytest_cache + From 15f81b055fa07804b20f330d8fc16343453c9bb6 Mon Sep 17 00:00:00 2001 From: ilai Date: Tue, 24 Oct 2023 10:07:02 +0300 Subject: [PATCH 57/64] [cli] Improve stop-on-error mechanism - Changed the name to "allow_failures", default false. - Print out the first encountered error --- src/resin_cli/cli.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/resin_cli/cli.py b/src/resin_cli/cli.py index 0d64ab34..4386c825 100644 --- a/src/resin_cli/cli.py +++ b/src/resin_cli/cli.py @@ -170,9 +170,12 @@ def new(index_name): ) @click.option("--batch-size", default=10, help="Number of documents to upload in each batch. Defaults to 10.") -@click.option("--stop-on-error/--no-stop-on-error", default=True, - help="Whether to stop when the first error arises. Defaults to True.") -def upsert(index_name: str, data_path: str, batch_size: int, stop_on_error: bool): +@click.option("--allow-failures/--dont-allow-failures", default=False, + help="On default, the upsert process will stop if any document fails to " + "be uploaded. " + "When set to True, the upsert process will continue on failure, as " + "long as less than 10% of the documents have failed to be uploaded.") +def upsert(index_name: str, data_path: str, batch_size: int, allow_failures: bool): if index_name is None: msg = ( "No index name provided. Please set --index-name or INDEX_NAME environment " @@ -228,26 +231,29 @@ def upsert(index_name: str, data_path: str, batch_size: int, stop_on_error: bool pbar = tqdm(total=len(data), desc="Upserting documents") failed_docs = [] + first_error = None for i in range(0, len(data), batch_size): batch = data[i:i + batch_size] try: kb.upsert(data) except Exception as e: - if stop_on_error or len(failed_docs) > len(data) // 10: + if allow_failures and len(failed_docs) < len(data) // 10: + failed_docs.extend([_.id for _ in batch]) + if first_error is None: + first_error = str(e) + else: msg = ( f"Failed to upsert data to index {kb.index_name}. " f"Underlying error: {e}" ) raise CLIError(msg) - else: - failed_docs.extend([_.id for _ in batch]) pbar.update(len(batch)) if failed_docs: msg = ( f"Failed to upsert the following documents to index {kb.index_name}: " - f"{failed_docs}" + f"{failed_docs}. The first encountered error was: {first_error}" ) raise CLIError(msg) From 14ab3078bdd035151fe9ce58a0ce6969bb101ec5 Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Tue, 24 Oct 2023 11:57:36 +0300 Subject: [PATCH 58/64] specify wheel and use poetry install for dev dependencies --- .github/workflows/pre-release-CI.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pre-release-CI.yml b/.github/workflows/pre-release-CI.yml index d272b94a..bc3c680e 100644 --- a/.github/workflows/pre-release-CI.yml +++ b/.github/workflows/pre-release-CI.yml @@ -1,7 +1,9 @@ name: Pre Release CI on: - workflow_dispatch: + push: + branches: + - pre-release-CI concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -38,11 +40,11 @@ jobs: - name: Install the wheel run: | - pip install dist/*.whl + pip install dist/pinecone_resin*.whl - name: Install tests dependencies run: | - pip install pytest pytest-html asyncio pytest-asyncio pytest-xdist pytest-mock + poetry install --no-root --only dev - name: Run tests run: pytest --html=report.html --self-contained-html tests/unit From 0e83e428ed9af0e81d3e4bdbf8a2541d204526a4 Mon Sep 17 00:00:00 2001 From: ilai Date: Tue, 24 Oct 2023 12:04:22 +0300 Subject: [PATCH 59/64] [cli] Improve error message Tell the user they can use --allow-failures --- src/resin_cli/cli.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/resin_cli/cli.py b/src/resin_cli/cli.py index 4386c825..5248ce71 100644 --- a/src/resin_cli/cli.py +++ b/src/resin_cli/cli.py @@ -1,4 +1,5 @@ import os +from typing import List, Optional import click import time @@ -230,8 +231,8 @@ def upsert(index_name: str, data_path: str, batch_size: int, allow_failures: boo abort=True) pbar = tqdm(total=len(data), desc="Upserting documents") - failed_docs = [] - first_error = None + failed_docs: List[str] = [] + first_error: Optional[str] = None for i in range(0, len(data), batch_size): batch = data[i:i + batch_size] try: @@ -244,7 +245,8 @@ def upsert(index_name: str, data_path: str, batch_size: int, allow_failures: boo else: msg = ( f"Failed to upsert data to index {kb.index_name}. " - f"Underlying error: {e}" + f"Underlying error: {e}\n" + f"You can allow partial failures by setting --allow-failures. " ) raise CLIError(msg) From 9d20f6668c3a2834fb2f6708ee057ee898cd14c1 Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Tue, 24 Oct 2023 12:08:58 +0300 Subject: [PATCH 60/64] use pip to install dev req --- .github/workflows/pre-release-CI.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pre-release-CI.yml b/.github/workflows/pre-release-CI.yml index bc3c680e..a406e3fc 100644 --- a/.github/workflows/pre-release-CI.yml +++ b/.github/workflows/pre-release-CI.yml @@ -42,9 +42,13 @@ jobs: run: | pip install dist/pinecone_resin*.whl - - name: Install tests dependencies + - name: Create dev requirements file run: | - poetry install --no-root --only dev + poetry export -f requirements.txt --without-hashes --only dev -o only-dev.txt + + - name: Install dev requirements + run: | + pip install -r only-dev.txt - name: Run tests run: pytest --html=report.html --self-contained-html tests/unit From e991fc7437a1dc99a41bdfb2ab5083c62c8296a8 Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Tue, 24 Oct 2023 12:18:25 +0300 Subject: [PATCH 61/64] dispatch manually --- .github/workflows/pre-release-CI.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/pre-release-CI.yml b/.github/workflows/pre-release-CI.yml index a406e3fc..9a09d046 100644 --- a/.github/workflows/pre-release-CI.yml +++ b/.github/workflows/pre-release-CI.yml @@ -1,9 +1,7 @@ name: Pre Release CI on: - push: - branches: - - pre-release-CI + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} From 621dc0302905cb17f690fa89798a7b0a74636a5e Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Tue, 24 Oct 2023 12:21:19 +0300 Subject: [PATCH 62/64] add py version to run name --- .github/workflows/pre-release-CI.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pre-release-CI.yml b/.github/workflows/pre-release-CI.yml index 9a09d046..50df3692 100644 --- a/.github/workflows/pre-release-CI.yml +++ b/.github/workflows/pre-release-CI.yml @@ -7,9 +7,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true -jobs: +jobs:git build-and-test: - name: Build & Test on ${{ matrix.os }} + name: Build & Test on ${{ matrix.os }}-py${{ matrix.python-version }} runs-on: ${{ matrix.os }} strategy: matrix: From f685c79e9dbebc2ff2e78e9905b0989ee16b1e92 Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Mon, 23 Oct 2023 16:54:59 +0300 Subject: [PATCH 63/64] resolve mypy --- src/resin_cli/cli.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/resin_cli/cli.py b/src/resin_cli/cli.py index 5248ce71..5a8b208b 100644 --- a/src/resin_cli/cli.py +++ b/src/resin_cli/cli.py @@ -52,7 +52,10 @@ def check_service_health(url: str): raise CLIError(msg) except requests.exceptions.HTTPError as e: - error = e.response.json().get("detail", None) or e.response.text + if e.response is not None: + error = e.response.json().get("detail", None) or e.response.text + else: + error = str(e) msg = ( f"Resin service on {url} is not healthy, failed with error: {error}" ) From ffa126009e173a433eb6e851c20a13949abb3889 Mon Sep 17 00:00:00 2001 From: Amnon Catav Date: Tue, 24 Oct 2023 12:34:28 +0300 Subject: [PATCH 64/64] Update src/resin_cli/cli.py Co-authored-by: igiloh-pinecone <118673156+igiloh-pinecone@users.noreply.github.com> --- src/resin_cli/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resin_cli/cli.py b/src/resin_cli/cli.py index aba635b7..7a87f78a 100644 --- a/src/resin_cli/cli.py +++ b/src/resin_cli/cli.py @@ -52,7 +52,7 @@ def check_service_health(url: str): except requests.exceptions.HTTPError as e: if e.response is not None: - error = e.response.json().get("detail", "") or e.response.text + error = e.response.json().get("detail", None) or e.response.text else: error = str(e) msg = (