diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index d56701e6..14d2b6ea 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: - python-version: ['3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v3 # Updated to the latest version diff --git a/README.md b/README.md index 2ee5a73b..14df37c7 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ [![](https://dcbadge.vercel.app/api/server/zt2mTPcu?compact=true&style=flat)](https://discord.gg/zt2mTPcu) -### ⚡ The PyTorch Library for Large Language Model Applications ⚡ +### ⚡ The Lightning Library for Large Language Model Applications ⚡ *LightRAG* helps developers with both building and optimizing *Retriever-Agent-Generator* pipelines. It is *light*, *modular*, and *robust*, with a 100% readable codebase. @@ -20,7 +20,7 @@ It is *light*, *modular*, and *robust*, with a 100% readable codebase. # Why LightRAG? -LLMs are like water; they can almost do anything, from GenAI applications such as chatbots, translation, summarization, code generation, and autonomous agents to classical NLP tasks like text classification and named entity recognition. They interact with the world beyond the model’s internal knowledge via retrievers, memory, and tools (function calls). Each use case is unique in its data, business logic, and user experience. +LLMs are like water; they can be shaped into anything, from GenAI applications such as chatbots, translation, summarization, code generation, and autonomous agents to classical NLP tasks like text classification and named entity recognition. They interact with the world beyond the model’s internal knowledge via retrievers, memory, and tools (function calls). Each use case is unique in its data, business logic, and user experience. Because of this, no library can provide out-of-the-box solutions. Users must build toward their own use case. This requires the library to be modular, robust, and have a clean, readable codebase. The only code you should put into production is code you either 100% trust or are 100% clear about how to customize and iterate. @@ -240,7 +240,7 @@ LightRAG full documentation available at [lightrag.sylph.ai](https://lightrag.sy ```bibtex @software{Yin2024LightRAG, author = {Li Yin}, - title = {{LightRAG: The PyTorch Library for Large Language Model (LLM) Applications}}, + title = {{LightRAG: The Lightning Library for Large Language Model (LLM) Applications}}, month = {7}, year = {2024}, doi = {10.5281/zenodo.12639531}, diff --git a/developer_notes/react_note.py b/developer_notes/react_note.py new file mode 100644 index 00000000..f5d68979 --- /dev/null +++ b/developer_notes/react_note.py @@ -0,0 +1,74 @@ +from lightrag.components.agent import ReActAgent +from lightrag.core import Generator, ModelClientType, ModelClient +from lightrag.utils import setup_env + +setup_env() + + +# Define tools +def multiply(a: int, b: int) -> int: + """ + Multiply two numbers. + """ + return a * b + + +def add(a: int, b: int) -> int: + """ + Add two numbers. + """ + return a + b + + +def divide(a: float, b: float) -> float: + """ + Divide two numbers. + """ + return float(a) / b + + +llama3_model_kwargs = { + "model": "llama3-70b-8192", # llama3 70b works better than 8b here. + "temperature": 0.0, +} +gpt_model_kwargs = { + "model": "gpt-3.5-turbo", + "temperature": 0.0, +} + + +def test_react_agent(model_client: ModelClient, model_kwargs: dict): + tools = [multiply, add, divide] + queries = [ + "What is the capital of France? and what is 465 times 321 then add 95297 and then divide by 13.2?", + "Give me 5 words rhyming with cool, and make a 4-sentence poem using them", + ] + # define a generator without tools for comparison + + generator = Generator( + model_client=model_client, + model_kwargs=model_kwargs, + ) + + react = ReActAgent( + max_steps=6, + add_llm_as_fallback=True, + tools=tools, + model_client=model_client, + model_kwargs=model_kwargs, + ) + # print(react) + + for query in queries: + print(f"Query: {query}") + agent_response = react.call(query) + llm_response = generator.call(prompt_kwargs={"input_str": query}) + print(f"Agent response: {agent_response}") + print(f"LLM response: {llm_response}") + print("") + + +if __name__ == "__main__": + test_react_agent(ModelClientType.GROQ(), llama3_model_kwargs) + # test_react_agent(ModelClientType.OPENAI(), gpt_model_kwargs) + print("Done") diff --git a/docs/Makefile b/docs/Makefile index 014ee19f..d763e798 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -21,7 +21,8 @@ help: apidoc: @sphinx-apidoc -o $(APIDOCOUTDIR)/core ../lightrag/lightrag/core --separate --force - @sphinx-apidoc -o $(APIDOCOUTDIR)/components ../lightrag/lightrag/components --separate --force --templatedir=$(SOURCEDIR)/_templates + @sphinx-apidoc -o $(APIDOCOUTDIR)/components ../lightrag/lightrag/components --separate --force +#--templatedir=$(SOURCEDIR)/_templates @sphinx-apidoc -o $(APIDOCOUTDIR)/eval ../lightrag/lightrag/eval --separate --force @sphinx-apidoc -o $(APIDOCOUTDIR)/optim ../lightrag/lightrag/optim --separate --force @sphinx-apidoc -o $(APIDOCOUTDIR)/utils ../lightrag/lightrag/utils --separate --force diff --git a/docs/source/_static/images/LightRAG_dataflow.png b/docs/source/_static/images/LightRAG_dataflow.png index b385553c..88444352 100644 Binary files a/docs/source/_static/images/LightRAG_dataflow.png and b/docs/source/_static/images/LightRAG_dataflow.png differ diff --git a/docs/source/_static/images/query_1.png b/docs/source/_static/images/query_1.png new file mode 100644 index 00000000..53e4d6d2 Binary files /dev/null and b/docs/source/_static/images/query_1.png differ diff --git a/docs/source/_static/images/query_2.png b/docs/source/_static/images/query_2.png new file mode 100644 index 00000000..b27ce8a0 Binary files /dev/null and b/docs/source/_static/images/query_2.png differ diff --git a/docs/source/apis/components/index.rst b/docs/source/apis/components/index.rst index cf125484..4b31f774 100644 --- a/docs/source/apis/components/index.rst +++ b/docs/source/apis/components/index.rst @@ -31,7 +31,6 @@ Retriever ~~~~~~~~~~~~~~~~~~~~ .. autosummary:: - :nosignatures: components.retriever.bm25_retriever components.retriever.faiss_retriever @@ -46,7 +45,6 @@ Output Parsers ~~~~~~~~~~~~~~~~~~~~ .. autosummary:: - :nosignatures: components.output_parsers.outputs @@ -54,7 +52,6 @@ Agent ~~~~~~~~~~~~~~~~~~~~ .. autosummary:: - :nosignatures: components.agent.react @@ -62,7 +59,6 @@ Data Process ~~~~~~~~~~~~~~~~~~~~ .. autosummary:: - :nosignatures: components.data_process.text_splitter @@ -73,7 +69,6 @@ Memory ~~~~~~~~~~~~~~~~~~~~ .. autosummary:: - :nosignatures: components.memory.memory @@ -81,7 +76,6 @@ Reasoning ~~~~~~~~~~~~~~~~~~~~ .. autosummary:: - :nosignatures: components.reasoning.chain_of_thought diff --git a/docs/source/apis/index.rst b/docs/source/apis/index.rst index 362cfcc2..d3adfe41 100644 --- a/docs/source/apis/index.rst +++ b/docs/source/apis/index.rst @@ -104,10 +104,11 @@ Utils .. autosummary:: utils.logger + utils.setup_env + utils.lazy_import utils.serialization utils.config utils.registry - utils.setup_env .. toctree:: diff --git a/docs/source/developer_notes/agent.rst b/docs/source/developer_notes/agent.rst index 0c9b6834..90e46e91 100644 --- a/docs/source/developer_notes/agent.rst +++ b/docs/source/developer_notes/agent.rst @@ -7,20 +7,451 @@ Agent -- Franklin and Graesser (1997) -Agents are LLM-based and themselves belong to another popular family of LLM applications besides of the well-known RAGs. -The key on Agents are their ability to reasoning, plannning, and acting via accessible tools. -In LightRAG, agents are simply a generator which can use tools, take multiple steps(sequential or parallel ) to complete a user query. + +Alongside the well-known RAGs, agents [1]_ are another popular family of LLM applications. +What makes agents stand out is their ability to reason, plan, and act via accessible tools. +When it comes to implementation, LightRAG has simplified it down to a generator that can use tools, taking multiple steps (sequential or parallel) to complete a user query. + + + +Design +---------------- +We will first introduce ReAct [2]_, a general paradigm for building agents with a sequential of interleaving thought, action, and observation steps. + +- **Thought**: The reasoning behind taking an action. +- **Action**: The action to take from a predefined set of actions. In particular, these are the tools/functional tools we have introduced in :doc:`tools`. +- **Observation**: The simplest scenario is the execution result of the action in string format. To be more robust, this can be defined in any way that provides the right amount of execution information for the LLM to plan the next step. + +Prompt and Data Models +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +:const:`DEFAULT_REACT_AGENT_SYSTEM_PROMPT` is the default prompt for React agent's LLM planner. +We can categorize the prompt template into four parts: + +1. Task description + +This part is the overall role setup and task description for the agent. + +.. code-block:: python + + task_desc = r"""You are a helpful assistant. + Answer the user's query using the tools provided below with minimal steps and maximum accuracy. + + Each step you will read the previous Thought, Action, and Observation(execution result of the action) and then provide the next Thought and Action.""" + +2. Tools, output format, and example + + + +This part of the template is exactly the same as how we were calling functions in the :doc:`tools`. +The ``output_format_str`` is generated by ``FunctionExpression`` via ``JsonOutputParser``. +It includes the actual output format and examples of a list of ``FunctionExpression`` instances. +We use ``thought`` and ``action`` fields of the ``FunctionExpression`` as the agent's response. + + +.. code-block:: python + + tools = r"""{% if tools %} + + {% for tool in tools %} + {{ loop.index }}. + {{tool}} + ------------------------ + {% endfor %} + + {% endif %} + {{output_format_str}}""" + + +3. Task specification to teach the planner how to "think". + + +We provide more detailed instruction to ensure the agent will always end with 'finish' action to complete the task. +Additionally, we teach it how to handle simple queries and complex queries. + +* For simple queries, we instruct the agent to finish with as few steps as possible. +* For complex queries, we teach the agent a 'divide-and-conquer' strategy to solve the query step by step. + +.. code-block:: python + + task_spec = r""" + - For simple queries: Directly call the ``finish`` action and provide the answer. + - For complex queries: + - Step 1: Read the user query and potentially divide it into subqueries. And get started with the first subquery. + - Call one available tool at a time to solve each subquery/subquestion. \ + - At step 'finish', join all subqueries answers and finish the task. + Remember: + - Action must call one of the above tools with name. It can not be empty. + - You will always end with 'finish' action to finish the task. The answer can be the final answer or failure message. + """ + +We put all these three parts together to be within the ```` tag. + +4. Agent step history. + + + +We use :class:`StepOutput` to record the agent's step history, including: + +- ``action``: This will be the ``FunctionExpression`` instance predicted by the agent. +- ``observation``: The execution result of the action. + +In particular, we format the steps history after the user query as follows: + +.. code-block:: python + + step_history = r"""User query: + {{ input_str }} + {# Step History #} + {% if step_history %} + + {% for history in step_history %} + Step {{ loop.index }}. + "Thought": "{{history.action.thought}}", + "Action": "{{history.action.action}}", + "Observation": "{{history.observation}}" + ------------------------ + {% endfor %} + + {% endif %} + You:""" + + +Tools +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +In addition to the tools provided by users, by default, we add a new tool named ``finish`` to allow the agent to stop and return the final answer. + +.. code-block:: python + + def finish(answer: str) -> str: + """Finish the task with answer.""" + return answer + +Simply returning a string might not fit all scenarios, and we might consider allowing users to define their own finish function in the future for more complex cases. + +Additionally, since the provided tools cannot always solve user queries, we allow users to configure if an LLM model should be used to solve a subquery via the ``add_llm_as_fallback`` parameter. +This LLM will use the same model client and model arguments as the agent's planner. Here is our code to specify the fallback LLM tool: + + + +.. code-block:: python + + _additional_llm_tool = ( + Generator(model_client=model_client, model_kwargs=model_kwargs) + if self.add_llm_as_fallback + else None + ) + + def llm_tool(input: str) -> str: + """I answer any input query with llm's world knowledge. Use me as a fallback tool or when the query is simple.""" + # use the generator to answer the query + try: + output: GeneratorOutput = _additional_llm_tool( + prompt_kwargs={"input_str": input} + ) + response = output.data if output else None + return response + except Exception as e: + log.error(f"Error using the generator: {e}") + print(f"Error using the generator: {e}") + + return None + + +React Agent +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +We define the class :class:`ReActAgent` to put everything together. +It will orchestrate two components: + +- ``planner``: A ``Generator`` that works with a ``JsonOutputParser`` to parse the output format and examples of the function calls using ``FunctionExpression``. +- ``ToolManager``: Manages a given list of tools, the finish function, and the LLM tool. It is responsible for parsing and executing the functions. + +Additionally, it manages `step_history` as a list of ``StepOutput`` instances for the agent's internal state. + + +.. list-table:: + :header-rows: 1 + :widths: 70 40 + + * - **Name** + - **Description** + * - ``__init__(self, tools: List[Union[Callable, AsyncCallable, FunctionTool]] = [], max_steps: int = 10, add_llm_as_fallback: bool = True, examples: List[FunctionExpression] = [], *, model_client: ModelClient, model_kwargs: Dict = {})`` + - Initialize the `ReActAgent` with the specified tools, maximum steps, fallback option, examples, model client, and model arguments. + * - ``call(self, input: str, prompt_kwargs: Optional[Dict] = {}, model_kwargs: Optional[Dict] = {}) -> Any`` + - Prompt the agent with an input query and process the steps to generate a response. + +Agent In Action +------------------- + + +We will set up two sets of models, `llama3-70b-8192`` by Groq and `gpt-3.5-turbo`` by OpenAI, to test two queries. +For comparison, we will compare these with a vanilla LLM response without using the agent. +Here are the code snippets: + +.. code-block:: python + + from lightrag.components.agent import ReActAgent + from lightrag.core import Generator, ModelClientType, ModelClient + from lightrag.utils import setup_env + + setup_env() + + + # Define tools + def multiply(a: int, b: int) -> int: + """ + Multiply two numbers. + """ + return a * b + + def add(a: int, b: int) -> int: + """ + Add two numbers. + """ + return a + b + + def divide(a: float, b: float) -> float: + """ + Divide two numbers. + """ + return float(a) / b + + llama3_model_kwargs = { + "model": "llama3-70b-8192", # llama3 70b works better than 8b here. + "temperature": 0.0, + } + gpt_model_kwargs = { + "model": "gpt-3.5-turbo", + "temperature": 0.0, + } + + + def test_react_agent(model_client: ModelClient, model_kwargs: dict): + tools = [multiply, add, divide] + queries = [ + "What is the capital of France? and what is 465 times 321 then add 95297 and then divide by 13.2?", + "Give me 5 words rhyming with cool, and make a 4-sentence poem using them", + ] + # define a generator without tools for comparison + + generator = Generator( + model_client=model_client, + model_kwargs=model_kwargs, + ) + + react = ReActAgent( + max_steps=6, + add_llm_as_fallback=True, + tools=tools, + model_client=model_client, + model_kwargs=model_kwargs, + ) + # print(react) + + for query in queries: + print(f"Query: {query}") + agent_response = react.call(query) + llm_response = generator.call(prompt_kwargs={"input_str": query}) + print(f"Agent response: {agent_response}") + print(f"LLM response: {llm_response}") + print("") + +The structure of React, including the initialization arguments and two major components: ``tool_manager`` and ``planner``, is shown below. + +.. raw:: html + +
+ 
+            
+
+   ReActAgent(
+      max_steps=6, add_llm_as_fallback=True,
+      (tool_manager): ToolManager(Tools: [FunctionTool(fn: , async: False, definition: FunctionDefinition(func_name='multiply', func_desc='multiply(a: int, b: int) -> int\n\n    Multiply two numbers.\n    ', func_parameters={'type': 'object', 'properties': {'a': {'type': 'int'}, 'b': {'type': 'int'}}, 'required': ['a', 'b']})), FunctionTool(fn: , async: False, definition: FunctionDefinition(func_name='add', func_desc='add(a: int, b: int) -> int\n\n    Add two numbers.\n    ', func_parameters={'type': 'object', 'properties': {'a': {'type': 'int'}, 'b': {'type': 'int'}}, 'required': ['a', 'b']})), FunctionTool(fn: , async: False, definition: FunctionDefinition(func_name='divide', func_desc='divide(a: float, b: float) -> float\n\n    Divide two numbers.\n    ', func_parameters={'type': 'object', 'properties': {'a': {'type': 'float'}, 'b': {'type': 'float'}}, 'required': ['a', 'b']})), FunctionTool(fn: .llm_tool at 0x11384b740>, async: False, definition: FunctionDefinition(func_name='llm_tool', func_desc="llm_tool(input: str) -> str\nI answer any input query with llm's world knowledge. Use me as a fallback tool or when the query is simple.", func_parameters={'type': 'object', 'properties': {'input': {'type': 'str'}}, 'required': ['input']})), FunctionTool(fn: .finish at 0x11382fa60>, async: False, definition: FunctionDefinition(func_name='finish', func_desc='finish(answer: str) -> str\nFinish the task with answer.', func_parameters={'type': 'object', 'properties': {'answer': {'type': 'str'}}, 'required': ['answer']}))], Additional Context: {})
+      (planner): Generator(
+         model_kwargs={'model': 'llama3-70b-8192', 'temperature': 0.0},
+         (prompt): Prompt(
+            template: 
+            {# role/task description #}
+            You are a helpful assistant.
+            Answer the user's query using the tools provided below with minimal steps and maximum accuracy.
+            {# REACT instructions #}
+            Each step you will read the previous Thought, Action, and Observation(execution result of the action) and then provide the next Thought and Action.
+            {# Tools #}
+            {% if tools %}
+            
+            You available tools are:
+            {# tools #}
+            {% for tool in tools %}
+            {{ loop.index }}.
+            {{tool}}
+            ------------------------
+            {% endfor %}
+            
+            {% endif %}
+            {# output format and examples #}
+            {{output_format_str}}
+            
+            {# Specifications TODO: preference between the usage of llm tool vs the other tool #}
+            - For simple queries: Directly call the ``finish`` action and provide the answer.
+            - For complex queries:
+               - Step 1: Read the user query and potentially divide it into subqueries. And get started with the first subquery.
+               - Call one available tool at a time to solve each subquery/subquestion. \
+               - At step 'finish', join all subqueries answers and finish the task.
+            Remember:
+            - Action must call one of the above tools with name. It can not be empty.
+            - You will always end with 'finish' action to finish the task. The answer can be the final answer or failure message.
+            
+            
+            -----------------
+            User query:
+            {{ input_str }}
+            {# Step History #}
+            {% if step_history %}
+            
+            {% for history in step_history %}
+            Step {{ loop.index }}.
+            "Thought": "{{history.action.thought}}",
+            "Action": "{{history.action.action}}",
+            "Observation": "{{history.observation}}"
+            ------------------------
+            {% endfor %}
+            
+            {% endif %}
+            You:, prompt_kwargs: {'tools': ['func_name: multiply\nfunc_desc: "multiply(a: int, b: int) -> int\\n\\n    Multiply two numbers.\\n    "\nfunc_parameters:\n  type: object\n  properties:\n    a:\n      type: int\n    b:\n      type: int\n  required:\n  - a\n  - b\n', 'func_name: add\nfunc_desc: "add(a: int, b: int) -> int\\n\\n    Add two numbers.\\n    "\nfunc_parameters:\n  type: object\n  properties:\n    a:\n      type: int\n    b:\n      type: int\n  required:\n  - a\n  - b\n', 'func_name: divide\nfunc_desc: "divide(a: float, b: float) -> float\\n\\n    Divide two numbers.\\n    "\nfunc_parameters:\n  type: object\n  properties:\n    a:\n      type: float\n    b:\n      type: float\n  required:\n  - a\n  - b\n', "func_name: llm_tool\nfunc_desc: 'llm_tool(input: str) -> str\n\n  I answer any input query with llm''s world knowledge. Use me as a fallback tool\n  or when the query is simple.'\nfunc_parameters:\n  type: object\n  properties:\n    input:\n      type: str\n  required:\n  - input\n", "func_name: finish\nfunc_desc: 'finish(answer: str) -> str\n\n  Finish the task with answer.'\nfunc_parameters:\n  type: object\n  properties:\n    answer:\n      type: str\n  required:\n  - answer\n"], 'output_format_str': 'Your output should be formatted as a standard JSON instance with the following schema:\n```\n{\n    "thought": "Why the function is called (Optional[str]) (optional)",\n    "action": "FuncName() Valid function call expression. Example: \\"FuncName(a=1, b=2)\\" Follow the data type specified in the function parameters.e.g. for Type object with x,y properties, use \\"ObjectType(x=1, y=2) (str) (required)"\n}\n```\nExamples:\n```\n{\n    "thought": "I have finished the task.",\n    "action": "finish(answer=\\"final answer: \'answer\'\\")"\n}\n________\n```\n-Make sure to always enclose the JSON output in triple backticks (```). Please do not add anything other than valid JSON output!\n-Use double quotes for the keys and string values.\n-DO NOT mistaken the "properties" and "type" in the schema as the actual fields in the JSON output.\n-Follow the JSON formatting conventions.'}, prompt_variables: ['input_str', 'tools', 'step_history', 'output_format_str']
+         )
+         (model_client): GroqAPIClient()
+         (output_processors): JsonOutputParser(
+            data_class=FunctionExpression, examples=[FunctionExpression(thought='I have finished the task.', action='finish(answer="final answer: \'answer\'")')], exclude_fields=None, return_data_class=True
+            (output_format_prompt): Prompt(
+            template: Your output should be formatted as a standard JSON instance with the following schema:
+            ```
+            {{schema}}
+            ```
+            {% if example %}
+            Examples:
+            ```
+            {{example}}
+            ```
+            {% endif %}
+            -Make sure to always enclose the JSON output in triple backticks (```). Please do not add anything other than valid JSON output!
+            -Use double quotes for the keys and string values.
+            -DO NOT mistaken the "properties" and "type" in the schema as the actual fields in the JSON output.
+            -Follow the JSON formatting conventions., prompt_variables: ['example', 'schema']
+            )
+            (output_processors): JsonParser()
+         )
+      )
+   )
+            
+
+
+ +Now, let's run the test function to see the agent in action. + +.. code-block:: python + + test_react_agent(ModelClientType.GROQ, llama3_model_kwargs) + test_react_agent(ModelClientType.OPENAI, gpt_model_kwargs) + +Our agent will show the core steps for developers via colored printout, including input_query, steps, and the final answer. +The printout of the first query with llama3 is shown below (without the color here): + + +.. code-block:: python + + 2024-07-10 16:48:47 - [react.py:287:call] - input_query: What is the capital of France? and what is 465 times 321 then add 95297 and then divide by 13.2 + + 2024-07-10 16:48:48 - [react.py:266:_run_one_step] - Step 1: + StepOutput(step=1, action=FunctionExpression(thought="Let's break down the query into subqueries and start with the first one.", action='llm_tool(input="What is the capital of France?")'), function=Function(thought=None, name='llm_tool', args=[], kwargs={'input': 'What is the capital of France?'}), observation='The capital of France is Paris!') + _______ + + 2024-07-10 16:48:49 - [react.py:266:_run_one_step] - Step 2: + StepOutput(step=2, action=FunctionExpression(thought="Now, let's move on to the second subquery.", action='multiply(a=465, b=321)'), function=Function(thought=None, name='multiply', args=[], kwargs={'a': 465, 'b': 321}), observation=149265) + _______ + + 2024-07-10 16:48:49 - [react.py:266:_run_one_step] - Step 3: + StepOutput(step=3, action=FunctionExpression(thought="Now, let's add 95297 to the result.", action='add(a=149265, b=95297)'), function=Function(thought=None, name='add', args=[], kwargs={'a': 149265, 'b': 95297}), observation=244562) + _______ + + 2024-07-10 16:48:50 - [react.py:266:_run_one_step] - Step 4: + StepOutput(step=4, action=FunctionExpression(thought="Now, let's divide the result by 13.2.", action='divide(a=244562, b=13.2)'), function=Function(thought=None, name='divide', args=[], kwargs={'a': 244562, 'b': 13.2}), observation=18527.424242424244) + _______ + + 2024-07-10 16:48:50 - [react.py:266:_run_one_step] - Step 5: + StepOutput(step=5, action=FunctionExpression(thought="Now, let's combine the answers of both subqueries.", action='finish(answer="The capital of France is Paris! and the result of the mathematical operation is 18527.424242424244.")'), function=Function(thought=None, name='finish', args=[], kwargs={'answer': 'The capital of France is Paris! and the result of the mathematical operation is 18527.424242424244.'}), observation='The capital of France is Paris! and the result of the mathematical operation is 18527.424242424244.') + _______ + 2024-07-10 16:48:50 - [react.py:301:call] - answer: + The capital of France is Paris! and the result of the mathematical operation is 18527.424242424244. + +For the second query, the printout: + +.. code-block:: python + + 2024-07-10 16:48:51 - [react.py:287:call] - input_query: Give me 5 words rhyming with cool, and make a 4-sentence poem using them + 2024-07-10 16:48:52 - [react.py:266:_run_one_step] - Step 1: + StepOutput(step=1, action=FunctionExpression(thought="I need to find 5 words that rhyme with 'cool'.", action='llm_tool(input="What are 5 words that rhyme with \'cool\'?")'), function=Function(thought=None, name='llm_tool', args=[], kwargs={'input': "What are 5 words that rhyme with 'cool'?"}), observation='Here are 5 words that rhyme with "cool":\n\n1. Rule\n2. Tool\n3. Fool\n4. Pool\n5. School') + _______ + + 2024-07-10 16:49:00 - [react.py:266:_run_one_step] - Step 2: + StepOutput(step=2, action=FunctionExpression(thought='Now that I have the rhyming words, I need to create a 4-sentence poem using them.', action='llm_tool(input="Create a 4-sentence poem using the words \'rule\', \'tool\', \'fool\', \'pool\', and \'school\'.")'), function=Function(thought=None, name='llm_tool', args=[], kwargs={'input': "Create a 4-sentence poem using the words 'rule', 'tool', 'fool', 'pool', and 'school'."}), observation="Here is a 4-sentence poem using the words 'rule', 'tool', 'fool', 'pool', and 'school':\n\nIn the classroom, we learn to rule,\nWith a pencil as our trusty tool.\nBut if we're not careful, we can be a fool,\nAnd end up swimming in the school pool.") + _______ + + 2024-07-10 16:49:12 - [react.py:266:_run_one_step] - Step 3: + StepOutput(step=3, action=FunctionExpression(thought='I have the poem, now I need to finish the task.', action='finish(answer="Here are 5 words that rhyme with \'cool\': rule, tool, fool, pool, school. Here is a 4-sentence poem using the words: In the classroom, we learn to rule, With a pencil as our trusty tool. But if we\'re not careful, we can be a fool, And end up swimming in the school pool.")'), function=Function(thought=None, name='finish', args=[], kwargs={'answer': "Here are 5 words that rhyme with 'cool': rule, tool, fool, pool, school. Here is a 4-sentence poem using the words: In the classroom, we learn to rule, With a pencil as our trusty tool. But if we're not careful, we can be a fool, And end up swimming in the school pool."}), observation="Here are 5 words that rhyme with 'cool': rule, tool, fool, pool, school. Here is a 4-sentence poem using the words: In the classroom, we learn to rule, With a pencil as our trusty tool. But if we're not careful, we can be a fool, And end up swimming in the school pool.") + _______ + + 2024-07-10 16:49:12 - [react.py:301:call] - answer: + Here are 5 words that rhyme with 'cool': rule, tool, fool, pool, school. Here is a 4-sentence poem using the words: In the classroom, we learn to rule, With a pencil as our trusty tool. But if we're not careful, we can be a fool, And end up swimming in the school pool. + +The comparison between the agent and the vanilla LLM response is shown below: + + +.. code-block:: + + Answer with agent: The capital of France is Paris! and the result of the mathematical operation is 18527.424242424244. + Answer without agent: GeneratorOutput(data="I'd be happy to help you with that!\n\nThe capital of France is Paris.\n\nNow, let's tackle the math problem:\n\n1. 465 × 321 = 149,485\n2. Add 95,297 to that result: 149,485 + 95,297 = 244,782\n3. Divide the result by 13.2: 244,782 ÷ 13.2 = 18,544.09\n\nSo, the answer is 18,544.09!", error=None, usage=None, raw_response="I'd be happy to help you with that!\n\nThe capital of France is Paris.\n\nNow, let's tackle the math problem:\n\n1. 465 × 321 = 149,485\n2. Add 95,297 to that result: 149,485 + 95,297 = 244,782\n3. Divide the result by 13.2: 244,782 ÷ 13.2 = 18,544.09\n\nSo, the answer is 18,544.09!", metadata=None) + + +For the second query, the comparison is shown below: + +.. code-block:: + + Answer with agent: Here are 5 words that rhyme with 'cool': rule, tool, fool, pool, school. Here is a 4-sentence poem using the words: In the classroom, we learn to rule, With a pencil as our trusty tool. But if we're not careful, we can be a fool, And end up swimming in the school pool. + Answer without agent: GeneratorOutput(data='Here are 5 words that rhyme with "cool":\n\n1. rule\n2. tool\n3. fool\n4. pool\n5. school\n\nAnd here\'s a 4-sentence poem using these words:\n\nIn the summer heat, I like to be cool,\nFollowing the rule, I take a dip in the pool.\nI\'m not a fool, I know just what to do,\nI grab my tool and head back to school.', error=None, usage=None, raw_response='Here are 5 words that rhyme with "cool":\n\n1. rule\n2. tool\n3. fool\n4. pool\n5. school\n\nAnd here\'s a 4-sentence poem using these words:\n\nIn the summer heat, I like to be cool,\nFollowing the rule, I take a dip in the pool.\nI\'m not a fool, I know just what to do,\nI grab my tool and head back to school.', metadata=None) + +The ReAct agent is particularly helpful for answering queries that require capabilities like computation or more complicated reasoning and planning. +However, using it on general queries might not be an overkill, as it might take more steps than necessary to answer the query. + +.. .. figure:: /_static/images/query_1.png +.. :align: center +.. :alt: DataClass +.. :width: 100% + +.. The internal terminal printout of the agent on the first query. + + +.. .. figure:: /_static/images/query_2.png +.. :align: center +.. :alt: DataClass +.. :width: 100% + +.. The internal terminal printout of the agent on the second query. .. admonition:: References :class: highlight - 1. A survey on large language model based autonomous agents: https://github.com/Paitesanshi/LLM-Agent-Survey - 2. ReAct: https://arxiv.org/abs/2210.03629 + .. [1] A survey on large language model based autonomous agents: https://github.com/Paitesanshi/LLM-Agent-Survey + .. [2] ReAct: https://arxiv.org/abs/2210.03629 .. admonition:: API References :class: highlight - - :class:`components.agent.react.ReactAgent` + - :class:`components.agent.react.ReActAgent` + - :class:`core.types.StepOutput` + - :const:`components.agent.react.DEFAULT_REACT_AGENT_SYSTEM_PROMPT` diff --git a/docs/source/developer_notes/base_data_class.rst b/docs/source/developer_notes/base_data_class.rst index 40e8d664..dcb3c6dd 100644 --- a/docs/source/developer_notes/base_data_class.rst +++ b/docs/source/developer_notes/base_data_class.rst @@ -300,7 +300,7 @@ The ``exclude`` parameter works the same across all methods. **DataClassFormatType** -For data class format, we have :class:``core.base_data_class.DataClassFormatType`` along with ``format_class_str`` method to specify the format type for the data format methods. +For data class format, we have :class:`DataClassFormatType` along with ``format_class_str`` method to specify the format type for the data format methods. .. code-block:: python diff --git a/docs/source/developer_notes/component.rst b/docs/source/developer_notes/component.rst index 026d5a71..9e465d4d 100644 --- a/docs/source/developer_notes/component.rst +++ b/docs/source/developer_notes/component.rst @@ -6,24 +6,30 @@ Component .. `Li Yin `_ -What you will learn? +.. What you will learn? + +.. 1. What is ``Component`` and why is it designed this way? +.. 2. How to use ``Component`` along with helper classes like ``FunComponent`` and ``Sequential``? + + +:ref:`Component` is to LLM task pipelines what `nn.Module` is to PyTorch models. +It is the base class for components such as ``Prompt``, ``ModelClient``, ``Generator``, ``Retriever`` in LightRAG. +Your task pipeline should also subclass from ``Component``. + -1. What is ``Component`` and why is it designed this way? -2. How to use ``Component`` along with helper classes like ``FunComponent`` and ``Sequential``? Design --------------------------------------- - :ref:`Component` is to LLM task pipelines what ``nn.Module`` is to PyTorch models. -It is the base class for components, such as ``Prompt``, ``ModelClient``, ``Generator``, ``Retriever`` in LightRAG. -Your task pipeline should subclass from ``Component`` too. Instead of working with ``Tensor`` and ``Parameter`` to train models with weights and biases, our component works with any data, ``Parameter`` that can be any data type for LLM in-context learning, from manual to auto prompt engineering. -We name it differently to avoid confusion and also for better compatibility with `PyTorch`. +Different from PyTorch's nn.Module, which works exclusively with Tensor and Parameter to train models with weights and biases, our component can work with different types of data, from a string or a list of strings to a list of :class:`Document`. + +.. `Parameter` that can be any data type for LLM in-context learning, from manual to auto prompt engineering. +Here is the comparison of writing a PyTorch model and a LightRAG task pipeline. -Here is the comparison of writing a PyTorch model and a LightRAG task component. -.. grid:: 2 +.. grid:: 1 :gutter: 1 .. grid-item-card:: PyTorch @@ -65,28 +71,49 @@ Here is the comparison of writing a PyTorch model and a LightRAG task component. def call(self, query: str) -> str: return self.doc(prompt_kwargs={"input_str": query}).data +As the fundamental building block in LLM task pipelines, the component is designed to serve five main purposes: + +1. **Standardize the interface for all components.** + This includes the `__init__` method, the `call` method for synchronous calls, the `acall` method for asynchronous calls, and the `__call__` method, which by default calls the `call` method. -As the foundamental building block in LLM task pipeline, the component is designed to serve five main purposes: +2. **Provide a unified way to visualize the structure of the task pipeline** + via the `__repr__` method. Subclasses can additionally add the `_extra_repr` method to include more information than the default `__repr__` method. -1. **Standarize the interface for all components.** This includes the `__init__` method, the `call` method for synchronous call, the `acall` method for asynchronous call, and the `__call__` which in default calls the `call` method. -2. **Provide a unified way to visualize the structure of the task pipeline** via `__repr__` method. And subclass can additional add `_extra_repr` method to add more information than the default `__repr__` method. -3. **Tracks, adds all subcomponents and parameters automatically and recursively** to assistant the building and optimizing process of the task pipeline. -4. **Manages the states and serialization**, with `state_dict` and `load_state_dict` methods in particular for parameters and `to_dict` method for serialization of all the states fall into the component's attributes, from subcomponents to parameters, to any other attributes of various data type. -5. **Make all components configurable from using `json` or `yaml` files**. This is especially useful for experimenting or building data processing pipelines. +3. **Track and add all subcomponents and parameters automatically and recursively** + to assist in the building and optimizing process of the task pipeline. -These features are key to keep LightRAG pipeline transparent, flexible, and easy to use. +4. **Manage the states and serialization**, + with `state_dict` and `load_state_dict` methods specifically for parameters, and the `to_dict` method for serialization of all states within the component's attributes, from subcomponents to parameters, to any other attributes of various data types. + +5. **Make all components configurable using `json` or `yaml` files**. + This is especially useful for experimenting or building data processing pipelines. + +These features are key to keeping the LightRAG pipeline transparent, flexible, and easy to use. By subclassing from the `Component` class, you will get most of these features out of the box. +.. As the foundamental building block in LLM task pipeline, the component is designed to serve five main purposes: + +.. 1. **Standarize the interface for all components.** This includes the `__init__` method, the `call` method for synchronous call, the `acall` method for asynchronous call, and the `__call__` which in default calls the `call` method. +.. 2. **Provide a unified way to visualize the structure of the task pipeline** via `__repr__` method. And subclass can additional add `_extra_repr` method to add more information than the default `__repr__` method. +.. 3. **Tracks, adds all subcomponents and parameters automatically and recursively** to assistant the building and optimizing process of the task pipeline. +.. 4. **Manages the states and serialization**, with `state_dict` and `load_state_dict` methods in particular for parameters and `to_dict` method for serialization of all the states fall into the component's attributes, from subcomponents to parameters, to any other attributes of various data type. +.. 5. **Make all components configurable from using `json` or `yaml` files**. This is especially useful for experimenting or building data processing pipelines. + +.. These features are key to keep LightRAG pipeline transparent, flexible, and easy to use. +.. By subclassing from the `Component` class, you will get most of these features out of the box. + + Component in Action --------------------------------------- -.. Transparency -.. ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + In this note, we are creating an AI doctor to answer medical questions. Run the ``DocQA`` on a query: + .. code-block:: python doc = DocQA() @@ -133,6 +160,7 @@ Configure from file As the above example shows, we added subcomponent via attributes. We can also use methods to add more subcomponnents or parameters. + .. code-block:: python from lightrag.core.parameter import Parameter @@ -141,8 +169,12 @@ We can also use methods to add more subcomponnents or parameters. # list all parameters for param in doc.named_parameters(): print(param) - # output - # ('demo', Parameter: demo) + +The output: + +.. code-block:: + + ('demo', Parameter: demo) You can easily save the detailed states: @@ -152,21 +184,25 @@ You can easily save the detailed states: save_json(doc.to_dict(), "doc.json") +To add even more flexibility, we provide :class:`FunComponent` and :class:`Sequential` for more advanced use cases. -To adds even more flexibility, we provide :class:`core.component.FunComponent` and :class:`core.component.Sequential` for more advanced use cases. Searalization and deserialization ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -We provide ``is_pickable`` method to check if the component is pickable. -And any of your component, it is a good practise to ensure it is pickable. +We provide the ``is_pickable`` method to check if the component is pickable. +It is good practice to ensure that any of your components are pickable. + + + + FunComponent -------------- - Use :func:`core.component.fun_to_component` as a decorator to convert any function to a Component with its unique class name. + Use :func:`fun_to_component` as a decorator to convert any function to a Component with its unique class name. -:class:`core.component.FunComponent` is a subclass of :class:`core.component.Component` that allows you to define a component with a function. +:class:`FunComponent` is a subclass of :class:`Component` that allows you to define a component with a function. You can directly use this class as: .. code-block:: python @@ -180,16 +216,21 @@ You can directly use this class as: print(fun_component(1)) print(type(fun_component)) - # output: - # 2 - # +The printout: + +.. code-block:: + 2 + -We also have :func:`core.component.fun_to_component` to convert a function to a FunComponent via decorator or directly call the function. + + +We also have :func:`fun_to_component` to convert a function to a `FunComponent` via a decorator or by directly calling the function. This approach gives you a unique component converted from the function name. Via direct call: + .. code-block:: python from lightrag.core.component import fun_to_component @@ -198,12 +239,17 @@ Via direct call: print(fun_component(1)) print(type(fun_component)) - # output: - # 2 - # +The output: + +.. code-block:: + + 2 + -Via decorator will be even more convenient to have a component from a function: + + +Using a decorator is an even more convenient way to create a component from a function: .. code-block:: python @@ -220,8 +266,12 @@ Via decorator will be even more convenient to have a component from a function: Sequential -------------- -We have :class:`core.component.Sequential` class to PyTorch's ``nn.Sequential`` class. This is especially useful to chain together components in a sequence. Much like the concept of ``chain`` or ``pipeline`` in other LLM libraries. -Let's put the FunComponent and DocQA together in a sequence: + + + +We have the :class:`Sequential` class, which is similar to PyTorch's ``nn.Sequential`` class. +This is especially useful for chaining together components in a sequence, much like the concept of ``chain`` or ``pipeline`` in other LLM libraries. +Let's put the `FunComponent`` and `DocQA`` together in a sequence: .. code-block:: python @@ -236,9 +286,12 @@ Let's put the FunComponent and DocQA together in a sequence: query = "What is the best treatment for headache?" print(seq(query)) -We automatically enhance users' queries before passing them to the DocQA component. +We automatically enhance users' queries before passing them to the `DocQA` component. The output is: + + + .. code-block:: 1. Over-the-counter pain relievers like acetaminophen, ibuprofen, or aspirin @@ -269,4 +322,4 @@ The structure of the sequence using ``print(seq)``: - :func:`core.component.fun_to_component` -We will have more advanced use cases in the upcoming tutorials. +We will cover more advanced use cases in the upcoming tutorials. diff --git a/docs/source/developer_notes/index.rst b/docs/source/developer_notes/index.rst index 18987344..79b37ca4 100644 --- a/docs/source/developer_notes/index.rst +++ b/docs/source/developer_notes/index.rst @@ -41,11 +41,30 @@ We have a clear :doc:`lightrag_design_philosophy`, which results in this :doc:`c class_hierarchy +Introduction +------------------- + + +:ref:`Component` is to LLM task pipelines what `nn.Module` is to PyTorch models. +An LLM task pipeline in LightRAG mainly consists of components, such as a `Prompt`, `ModelClient`, `Generator`, `Retriever`, `Agent`, or any other custom components. +This pipeline can be `Sequential` or a Directed Acyclic Graph (DAG) of components. +A `Prompt` will work with `DataClass` to ease data interaction with the LLM model. +A `Retriever` will work with databases to retrieve context and overcome the hallucination and knowledge limitations of LLM, following the paradigm of Retrieval-Augmented Generation (RAG). +An `Agent` will work with tools and an LLM planner for enhanced ability to reason, plan, and act on real-world tasks. + +Additionally, what shines in LightRAG is that all orchestrator components, like `Retriever`, `Embedder`, `Generator`, and `Agent`, are model-agnostic. +You can easily make each component work with different models from different providers by switching out the `ModelClient` and its `model_kwargs`. + + +We will introduce the libraries starting from the core base classes, then move to the RAG essentials, and finally to the agent essentials. +With these building blocks, we will further introduce optimizing, where the optimizer uses building blocks such as Generator for auto-prompting and retriever for dynamic few-shot in-context learning (ICL). Building ------------------- + + Base classes ~~~~~~~~~~~~~~~~~~~~~~ Code path: :ref:`lightrag.core `. diff --git a/docs/source/developer_notes/lightrag_design_philosophy.rst b/docs/source/developer_notes/lightrag_design_philosophy.rst index e7242a0d..faee40f1 100644 --- a/docs/source/developer_notes/lightrag_design_philosophy.rst +++ b/docs/source/developer_notes/lightrag_design_philosophy.rst @@ -50,7 +50,7 @@ The above principles are distilled from our experiences and continuous learning **Developers are the ultimate heroes** -LLMs are like `water`, they can almost do anything, from GenAI applications such as `chatbot`, `translation`, `summarization`, `code generation`, `autonomous agent` to classical NLP tasks like `text classification`, and `named entity recognition`. +LLMs are like `water`, they can be shaped into anything, from GenAI applications such as `chatbot`, `translation`, `summarization`, `code generation`, `autonomous agent` to classical NLP tasks like `text classification`, and `named entity recognition`. They interact with the world beyond the model's internal knowledge via `retriever`, `memory`, and `tools` (`function calls`). Each use case is unique in its data, its business logic, and its unique user experience. diff --git a/docs/source/developer_notes/model_client.rst b/docs/source/developer_notes/model_client.rst index ce49548c..7bb83b34 100644 --- a/docs/source/developer_notes/model_client.rst +++ b/docs/source/developer_notes/model_client.rst @@ -6,14 +6,16 @@ ModelClient .. `Li Yin `_ -What you will learn? +.. What you will learn? -1. What is ``ModelClient`` and why is it designed this way? -2. How to intergrate your own ``ModelClient``? -3. How to use ``ModelClient`` directly? +.. 1. What is ``ModelClient`` and why is it designed this way? +.. 2. How to intergrate your own ``ModelClient``? +.. 3. How to use ``ModelClient`` directly? + + +:ref:`ModelClient` is the standardized protocol and base class for all model inference SDKs (either via APIs or local) to communicate with LightRAG internal components. +Therefore, by switching out the ``ModelClient`` in a ``Generator``, ``Embedder``, or ``Retriever`` (those components that take models), you can make these functional components model-agnostic. -:ref:`ModelClient` is the standardized protocol and base class for all model inference SDKs (either via APIs or local) to communicate with LightRAG internal components/classes. -Because so, by switching off ``ModelClient`` in a ``Generator`` or ``Embedder`` component, you can make your prompt or ``Retriever`` model-agnostic. .. figure:: /_static/images/model_client.png @@ -21,17 +23,20 @@ Because so, by switching off ``ModelClient`` in a ``Generator`` or ``Embedder`` :alt: ModelClient :width: 400px - The interface to internal components in LightRAG + The bridge between all model inference SDKs and internal components in LightRAG .. note:: - All users are encouraged to customize your own ``ModelClient`` whenever you need to do so. You can refer our code in ``components.model_client`` dir. + All users are encouraged to customize their own ``ModelClient`` whenever needed. You can refer to our code in ``components.model_client`` directory. + Model Inference SDKs ------------------------ -With cloud API providers like OpenAI, Groq, Anthropic, it often comes with a `sync` and an `async` client via their SDKs. + +With cloud API providers like OpenAI, Groq, and Anthropic, it often comes with a `sync` and an `async` client via their SDKs. For example: + .. code-block:: python from openai import OpenAI, AsyncOpenAI @@ -42,128 +47,32 @@ For example: # sync call using APIs response = sync_client.chat.completions.create(...) -For local models, such as using `huggingface transformers`, you need to create this model inference SDKs yourself. -How you do this is highly flexible. Here is an example to use local embedding model (e.g. ``thenlper/gte-base``) as a model (Refer :class:`components.model_client.transformers_client.TransformerEmbedder` for details). +For local models, such as using `huggingface transformers`, you need to create these model inference SDKs yourself. +How you do this is highly flexible. +Here is an example of using a local embedding model (e.g., ``thenlper/gte-base``) as a model (Refer to :class:`TransformerEmbedder` for details). It really is just normal model inference code. -.. code-block:: python - - from transformers import AutoTokenizer, AutoModel - - class TransformerEmbedder: - models: Dict[str, type] = {} - - def __init__(self, model_name: Optional[str] = "thenlper/gte-base"): - super().__init__() - - if model_name is not None: - self.init_model(model_name=model_name) - - @lru_cache(None) - def init_model(self, model_name: str): - try: - self.tokenizer = AutoTokenizer.from_pretrained(model_name) - self.model = AutoModel.from_pretrained(model_name) - # register the model - self.models[model_name] = self.model - - except Exception as e: - log.error(f"Error loading model {model_name}: {e}") - raise e - - def infer_gte_base_embedding( - self, - input=Union[str, List[str]], - tolist: bool = True, - ): - model = self.models.get("thenlper/gte-base", None) - if model is None: - # initialize the model - self.init_model("thenlper/gte-base") - - if isinstance(input, str): - input = [input] - # Tokenize the input texts - batch_dict = self.tokenizer( - input, max_length=512, padding=True, truncation=True, return_tensors="pt" - ) - outputs = model(**batch_dict) - embeddings = average_pool( - outputs.last_hidden_state, batch_dict["attention_mask"] - ) - # (Optionally) normalize embeddings - embeddings = F.normalize(embeddings, p=2, dim=1) - if tolist: - embeddings = embeddings.tolist() - return embeddings - - def __call__(self, **kwargs): - if "model" not in kwargs: - raise ValueError("model is required") - # load files and models, cache it for the next inference - model_name = kwargs["model"] - # inference the model - if model_name == "thenlper/gte-base": - return self.infer_gte_base_embedding(kwargs["input"]) - else: - raise ValueError(f"model {model_name} is not supported") - - ModelClient Protocol ----------------------------------------------------------------------------------------------------------- -A model client can be used to manage different types of models, we defined a ``ModelType`` to categorize the model type. +A model client can be used to manage different types of models, we defined a :class:`ModelType` to categorize the model type. .. code-block:: python class ModelType(Enum): EMBEDDER = auto() LLM = auto() + RERANKER = auto() UNDEFINED = auto() -We designed 6 abstract methods in the ``ModelClient`` class to be implemented by the subclass model type. -We will use :class:`components.model_client.OpenAIClient` along with the above ``TransformerEmbedder`` as examples. - -First, we offer two methods to initialize the model SDKs: - -.. code-block:: python - - def init_sync_client(self): - raise NotImplementedError( - f"{type(self).__name__} must implement _init_sync_client method" - ) - - def init_async_client(self): - raise NotImplementedError( - f"{type(self).__name__} must implement _init_async_client method" - ) +We designed 6 abstract methods in the `ModelClient` class that can be implemented by subclasses to integrate with different model inference SDKs. +We will use :class:`OpenAIClient` as the cloud API example and :class:`TransformersClient` along with the local inference code :class:`TransformerEmbedder` as an example for local model clients. -This is how `OpenAIClient` implements these methods along with ``__init__`` method: - -.. code-block:: python - class OpenAIClient(ModelClient): - - def __init__(self, api_key: Optional[str] = None): - - super().__init__() - self._api_key = api_key - self.sync_client = self.init_sync_client() - self.async_client = None # only initialize if the async call is called - - def init_sync_client(self): - api_key = self._api_key or os.getenv("OPENAI_API_KEY") - if not api_key: - raise ValueError("Environment variable OPENAI_API_KEY must be set") - return OpenAI(api_key=api_key) - - def init_async_client(self): - api_key = self._api_key or os.getenv("OPENAI_API_KEY") - if not api_key: - raise ValueError("Environment variable OPENAI_API_KEY must be set") - return AsyncOpenAI(api_key=api_key) +First, we offer two methods, `init_async_client` and `init_sync_client`, for subclasses to initialize the SDK client. +You can refer to :class:`OpenAIClient` to see how these methods, along with the `__init__` method, are implemented: This is how ``TransformerClient`` does the same thing: @@ -183,8 +92,7 @@ This is how ``TransformerClient`` does the same thing: def init_sync_client(self): return TransformerEmbedder() - -Second. we use `convert_inputs_to_api_kwargs` for subclass to convert LightRAG inputs into the `api_kwargs` (SDKs arguments). +Second, we use `convert_inputs_to_api_kwargs` for subclasses to convert LightRAG inputs into the `api_kwargs` (SDK arguments). .. code-block:: python @@ -228,6 +136,15 @@ This is how `OpenAIClient` implements this method: raise ValueError(f"model_type {model_type} is not supported") return final_model_kwargs +.. For embedding, as `Embedder` takes both `str` and `List[str]` as input, we need to convert the input to a list of strings. +.. For LLM, as `Generator` takes a `prompt_kwargs` (dict) and converts it into a single string, we need to convert the input to a list of messages. +.. For Rerankers, you can refer to :class:`CohereAPIClient` for an example. + + +For embedding, as ``Embedder`` takes both `str` and `List[str]` as input, we need to convert the input to a list of strings that is acceptable by the SDK. +For LLM, as ``Generator`` will takes a `prompt_kwargs`(dict) and convert it into a single string, thus we need to convert the input to a list of messages. +For Rerankers, you can refer to :class:`CohereAPIClient` for an example. + This is how ``TransformerClient`` does the same thing: .. code-block:: python @@ -245,37 +162,15 @@ This is how ``TransformerClient`` does the same thing: else: raise ValueError(f"model_type {model_type} is not supported") -In addition, you can add any method that parse the SDK specific output to a format compatible with LightRAG components. -Typically an LLM needs to use `parse_chat_completion` to parse the completion to texts and `parse_embedding_response` to parse the embedding response to a structure LightRAG components can understand. - -.. code-block:: python - - def parse_chat_completion(self, completion: Any) -> str: - raise NotImplementedError( - f"{type(self).__name__} must implement parse_chat_completion method" - ) +In addition, you can add any method that parses the SDK-specific output to a format compatible with LightRAG components. +Typically, an LLM needs to use `parse_chat_completion` to parse the completion to text and `parse_embedding_response` to parse the embedding response to a structure that LightRAG components can understand. +You can refer to :class:`OpenAIClient` for API embedding model integration and :class:`TransformersClient` for local embedding model integration. - def parse_embedding_response(self, response: Any) -> EmbedderOutput: - r"""Parse the embedding response to a structure LightRAG components can understand.""" - raise NotImplementedError( - f"{type(self).__name__} must implement parse_embedding_response method" - ) -You can refer to :class:`components.model_client.openai_client.OpenAIClient` for API embedding model integration and :class:`components.model_client.transformers_client.TransformersClient` for local embedding model integration. +Lastly, the `call` and `acall` methods are used to call model inference via their own arguments. +We encourage subclasses to provide error handling and retry mechanisms in these methods. -Then `call` and `acall` methods to call Model inference via their own arguments. -We encourage the subclass provides error handling and retry mechanism in these methods. - -.. code-block:: python - - def call(self, api_kwargs: Dict = {}, model_type: ModelType = ModelType.UNDEFINED): - raise NotImplementedError(f"{type(self).__name__} must implement _call method") - - async def acall( - self, api_kwargs: Dict = {}, model_type: ModelType = ModelType.UNDEFINED - ): - pass The `OpenAIClient` example: @@ -296,14 +191,19 @@ The `TransformerClient` example: def call(self, api_kwargs: Dict = {}, model_type: ModelType = ModelType.UNDEFINED): return self.sync_client(**api_kwargs) - -Our library currently integrated with 5 providers: OpenAI, Groq, Anthropic, Huggingface, and Google. +O +ur library currently integrates with six providers: OpenAI, Groq, Anthropic, Huggingface, Google, and Cohere. Please check out :ref:`ModelClient Integration`. + + Use ModelClient directly ----------------------------------------------------------------------------------------------------------- -Though ``ModelClient`` is often managed in a ``Generator`` or ``Embedder`` component, you can use it directly if you ever plan to write your own component. -Here is an example to use ``OpenAIClient`` directly, first on LLM model: + + +Though ``ModelClient`` is often managed in a ``Generator``, ``Embedder``, or ``Retriever`` component, you can use it directly if you plan to write your own component. +Here is an example of using ``OpenAIClient`` directly, first on an LLM model: + .. code-block:: python @@ -311,6 +211,8 @@ Here is an example to use ``OpenAIClient`` directly, first on LLM model: from lightrag.core.types import ModelType from lightrag.utils import setup_env + setup_env() + openai_client = OpenAIClient() query = "What is the capital of France?" @@ -361,6 +263,10 @@ The output will be: api_kwargs: {'model': 'text-embedding-3-small', 'dimensions': 8, 'encoding_format': 'float', 'input': ['What is the capital of France?', 'What is the capital of France?']} reponse_embedder_output: EmbedderOutput(data=[Embedding(embedding=[0.6175549, 0.24047995, 0.4509756, 0.37041178, -0.33437008, -0.050995983, -0.24366009, 0.21549304], index=0), Embedding(embedding=[0.6175549, 0.24047995, 0.4509756, 0.37041178, -0.33437008, -0.050995983, -0.24366009, 0.21549304], index=1)], model='text-embedding-3-small', usage=Usage(prompt_tokens=14, total_tokens=14), error=None, raw_response=None) + +.. TODO: add optional package introduction here + + .. admonition:: API reference :class: highlight @@ -370,3 +276,4 @@ The output will be: - :class:`components.model_client.groq_client.GroqAPIClient` - :class:`components.model_client.anthropic_client.AnthropicAPIClient` - :class:`components.model_client.google_client.GoogleGenAIClient` + - :class:`components.model_client.cohere_client.CohereAPIClient` diff --git a/docs/source/developer_notes/output_parsers.rst b/docs/source/developer_notes/output_parsers.rst index 314766a6..fc4f8167 100644 --- a/docs/source/developer_notes/output_parsers.rst +++ b/docs/source/developer_notes/output_parsers.rst @@ -1,7 +1,9 @@ Parser ============= -In this note, we will explain LightRAG parser and output parsers. +Parser is the `interpreter` of the LLM output. + + Context ---------------- @@ -21,7 +23,6 @@ It is an important step for the LLM applications to interact with the external w - to list to support multiple choice selection. - to json/yaml which will be extracted to dict, and optional further to data class instance to support support cases like function calls. -Parsing is the `interpreter` of the LLM output. Scope and Design ------------------ diff --git a/docs/source/index.rst b/docs/source/index.rst index 5c58109b..47da44ac 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -22,12 +22,12 @@ .. raw:: html -

⚡ The PyTorch Library for Large Language Model Applications ⚡

+

⚡ The Lightning Library for Large Language Model Applications ⚡

LightRAG helps developers with both building and optimizing Retriever-Agent-Generator pipelines.
- It is light, modular, and robust, with a 100% readable codebase. + It is light, modular, and robust, with a 100% readable codebase.

@@ -37,6 +37,26 @@ + + + + + + +.. and Customizability + + +Light +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +LightRAG shares similar design pattern as `PyTorch` for deep learning modeling. +We provide developers with fundamental building blocks of *100% clarity and simplicity*. + +- Only two fundamental but powerful base classes: `Component` for the pipeline and `DataClass` for data interaction with LLMs. +- A highly readable codebase and less than two levels of class inheritance. :doc:`developer_notes/class_hierarchy`. +- We maximize the library's tooling and prompting capabilities to minimize the reliance on LLM API features such as tools and JSON format. +- The result is a library with bare minimum abstraction, providing developers with *maximum customizability*. + + .. grid:: 1 :gutter: 1 @@ -94,23 +114,6 @@ async def acall(self, query): return await self.generator.acall({"input_str": query}) - - - - -.. and Customizability - - -Light -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -We provide developers with fundamental building blocks of *100% clarity and simplicity*. - -- Only two fundamental but powerful base classes: `Component` for the pipeline and `DataClass` for data interaction with LLMs. -- A highly readable codebase and less than two levels of class inheritance. :doc:`developer_notes/class_hierarchy`. -- We maximize the library's tooling and prompting capabilities to minimize the reliance on LLM API features such as tools and JSON format. -- The result is a library with bare minimum abstraction, providing developers with *maximum customizability*. - - .. - We use 10X less code than other libraries to achieve 10X more robustness and flexibility. .. - `Class Hierarchy Visualization `_ diff --git a/docs/source/insert_autosummary.py b/docs/source/insert_autosummary.py index 16d9fa64..4469bae2 100644 --- a/docs/source/insert_autosummary.py +++ b/docs/source/insert_autosummary.py @@ -54,6 +54,25 @@ def generate_rst_for_module(module_full_name, module, output_dir): content += f" {class_name}\n" content += "\n" + # Collect constants from __all__ + constants = [] + if hasattr(module, "__all__"): + all_members = getattr(module, "__all__") + for const_name in all_members: + const_value = getattr(module, const_name, None) + if ( + const_value is not None + and not inspect.isfunction(const_value) + and not inspect.isclass(const_value) + ): + constants.append((const_name, const_value)) + + if constants: + content += " .. rubric:: Constants\n\n" + for const_name, const_value in constants: + content += f" .. autodata:: {module_full_name}.{const_name}\n" + content += "\n" + with open(rst_filepath, "a") as rst_file: rst_file.write(content) diff --git a/lightrag/.gitignore b/lightrag/.gitignore index 799780fd..53607e8b 100644 --- a/lightrag/.gitignore +++ b/lightrag/.gitignore @@ -1 +1 @@ -test*.py +tests/log diff --git a/lightrag/CHANGELOG.md b/lightrag/CHANGELOG.md index d9f8a191..fd876f53 100644 --- a/lightrag/CHANGELOG.md +++ b/lightrag/CHANGELOG.md @@ -1,3 +1,15 @@ +## [0.0.0-beta.1] - 2024-07-10 + +### Added +- `DataClass`: add `__type_var_map__` in `data class schema` as the necessary step to support `Generic` in data class. +- Support Python `3.9`. + +### Fixed +- `ReAct` agent is fixed to be working with updates on the json output parser. +- `Add` error handling for using Lazy Import classes the wrong way, such as subclass. + + + ## [0.0.0-alpha.16] - 2024-07-08 ### Fixed diff --git a/lightrag/README.md b/lightrag/README.md index 0207856a..236a33dd 100644 --- a/lightrag/README.md +++ b/lightrag/README.md @@ -1,7 +1,7 @@ ![LightRAG Logo](https://raw.githubusercontent.com/SylphAI-Inc/LightRAG/main/docs/source/_static/images/LightRAG-logo-doc.jpeg) -### ⚡ The PyTorch Library for Large Language Model Applications ⚡ +### ⚡ The Lightning Library for Large Language Model Applications ⚡ *LightRAG* helps developers with both building and optimizing *Retriever-Agent-Generator (RAG)* pipelines. It is *light*, *modular*, and *robust*. @@ -97,7 +97,7 @@ LightRAG full documentation available at [lightrag.sylph.ai](https://lightrag.sy ```bibtex @software{Yin2024LightRAG, author = {Li Yin}, - title = {{LightRAG: The PyTorch Library for Large Language Model (LLM) Applications}}, + title = {{LightRAG: The Lightning Library for Large Language Model (LLM) Applications}}, month = {7}, year = {2024}, doi = {10.5281/zenodo.12639531}, diff --git a/lightrag/lightrag/components/agent/react.py b/lightrag/lightrag/components/agent/react.py index 6b37b6bd..3a769e6c 100644 --- a/lightrag/lightrag/components/agent/react.py +++ b/lightrag/lightrag/components/agent/react.py @@ -1,4 +1,4 @@ -"""Implementation of ReAct.""" +"""Implementation of ReAct Agent.""" from typing import List, Union, Callable, Optional, Any, Dict from copy import deepcopy @@ -23,7 +23,9 @@ log = logging.getLogger(__name__) -DEFAULT_REACT_AGENT_SYSTEM_PROMPT = r"""<> +__all__ = ["DEFAULT_REACT_AGENT_SYSTEM_PROMPT", "ReActAgent"] + +DEFAULT_REACT_AGENT_SYSTEM_PROMPT = r""" {# role/task description #} You are a helpful assistant. Answer the user's query using the tools provided below with minimal steps and maximum accuracy. @@ -41,10 +43,8 @@ {% endfor %} {% endif %} -{# output is always more robust to use json than string #} - +{# output format and examples #} {{output_format_str}} - {# Specifications TODO: preference between the usage of llm tool vs the other tool #} - For simple queries: Directly call the ``finish`` action and provide the answer. @@ -56,37 +56,23 @@ - Action must call one of the above tools with name. It can not be empty. - You will always end with 'finish' action to finish the task. The answer can be the final answer or failure message. -{#Examples can be here#} -{# Check if there are any examples #} -{% if examples %} - -{% for example in examples %} -{{ example }} -{% endfor %} - -{% endif %} - -<> + ----------------- -{% if input_str %} User query: {{ input_str }} -{% endif %} {# Step History #} {% if step_history %} {% for history in step_history %} Step {{ loop.index }}. -{ - "thought": "{{history.thought}}", - "action": "{{history.action.action}}", -} +"Thought": "{{history.action.thought}}", +"Action": "{{history.action.action}}", "Observation": "{{history.observation}}" ------------------------ {% endfor %} {% endif %} -""" +You:""" class ReActAgent(Component): @@ -138,12 +124,14 @@ def add(a: int, b: int) -> int: [1] https://arxiv.org/abs/2210.03629, published in Mar, 2023. """ + # TODO: allow users to pass in a few examples. Need to be a list of FunctionExpression instances. def __init__( self, # added arguments specifc to React tools: List[Union[Callable, AsyncCallable, FunctionTool]] = [], max_steps: int = 10, add_llm_as_fallback: bool = True, + examples: List[FunctionExpression] = [], *, # the following arguments are mainly for the planner model_client: ModelClient, @@ -164,7 +152,11 @@ def __init__( func=self._finish, answer="final answer: 'answer'", ) - output_parser = JsonOutputParser(data_class=ouput_data_class, example=example) + self._examples = examples + [example] + + output_parser = JsonOutputParser( + data_class=ouput_data_class, examples=self._examples, return_data_class=True + ) prompt_kwargs = { "tools": self.tool_manager.yaml_definitions, "output_format_str": output_parser.format_instructions(), @@ -217,7 +209,7 @@ def finish(answer: str) -> str: if self.add_llm_as_fallback: tools.append(llm_tool) tools.append(finish) - self.tool_manager = ToolManager(tools=tools) + self.tool_manager: ToolManager = ToolManager(tools=tools) def reset(self): r"""Reset the agent to start a new query.""" @@ -230,12 +222,10 @@ def _execute_action(self, action_step: StepOutput) -> Optional[StepOutput]: action = action_step.action try: - fun: Function = self.tool_manager.parse_function_call_expr(action) - result: FunctionOutput = self.tool_manager.execute_function(fun) + fun: Function = self.tool_manager.parse_func_expr(action) + result: FunctionOutput = self.tool_manager.execute_func(fun) # TODO: optimize the action_step - action_step.fun_name = fun.name - action_step.fun_args = fun.args - action_step.fun_kwargs = fun.kwargs + action_step.function = fun action_step.observation = result.output return action_step except Exception as e: @@ -246,10 +236,9 @@ def _execute_action(self, action_step: StepOutput) -> Optional[StepOutput]: def _run_one_step(self, step: int, prompt_kwargs: Dict, model_kwargs: Dict) -> str: """ - Run one step of the agent. + Run one step of the agent. Plan and execute the action for the step. """ - # step_history is the only per-query variable, and should not be controlled by the user - # add the step_history to the prompt_kwargs + step_output: StepOutput = StepOutput(step=step) prompt_kwargs["step_history"] = self.step_history log.debug( @@ -260,27 +249,28 @@ def _run_one_step(self, step: int, prompt_kwargs: Dict, model_kwargs: Dict) -> s response: GeneratorOutput = self.planner( prompt_kwargs=prompt_kwargs, model_kwargs=model_kwargs ) - step_output: StepOutput = None - try: - fun_expr: FunctionExpression = FunctionExpression.from_dict(response.data) - step_output = StepOutput( - step=step, thought=fun_expr.thought, action=fun_expr - ) - # print the func expr - log.debug(f"Step {step}: {fun_expr}") - - # execute the action - if step_output and step_output.action: - step_output = self._execute_action(step_output) - printc(f"Step {step}: \n{step_output}\n_______\n", color="blue") - else: - log.error(f"Failed to parse response for step {step}") - except Exception as e: - log.error(f"Error running step {step}: {e}") - if step_output is None: - step_output = StepOutput(step=step, thought="", action="") - else: - step_output.observation = f"Error running step {step}: {e}" + if response.error: + error_msg = f"Error planning step {step}: {response.error}" + step_output.observation = error_msg + log.error(error_msg) + else: + try: + fun_expr: FunctionExpression = response.data + step_output.action = fun_expr + # print the func expr + log.debug(f"Step {step}: {fun_expr}") + + # execute the action + if step_output and step_output.action: + step_output = self._execute_action(step_output) + printc(f"Step {step}: \n{step_output}\n_______\n", color="blue") + else: + log.error(f"Failed to parse response for step {step}") + except Exception as e: + error_msg = f"Error parsing response for step {step}: {e}" + step_output.observation = error_msg + log.error(error_msg) + self.step_history.append(step_output) return response @@ -300,8 +290,8 @@ def call( try: self._run_one_step(step, prompt_kwargs, model_kwargs) if ( - self.step_history[-1].fun_name - and self.step_history[-1].fun_name == "finish" + self.step_history[-1].function + and self.step_history[-1].function.name == "finish" ): break except Exception as e: @@ -314,112 +304,5 @@ def call( return answer def _extra_repr(self) -> str: - s = f"max_steps={self.max_steps}, add_llm_as_fallback={self.add_llm_as_fallback}" - s += super()._extra_repr() + s = f"max_steps={self.max_steps}, add_llm_as_fallback={self.add_llm_as_fallback}, " return s - - -if __name__ == "__main__": - from components.model_client import GroqAPIClient - from lightrag.core.types import ModelClientType - from lightrag.utils import setup_env - - setup_env() - - def multiply(a: int, b: int) -> int: - """ - Multiply two numbers. - """ - return a * b - - def add(a: int, b: int) -> int: - """ - Add two numbers. - """ - return a + b - - def divide(a: float, b: float) -> float: - """ - Divide two numbers. - """ - return float(a) / b - - def search(query: str) -> str: - """ - Search the web for the given query. - """ - return "python programming is a great way to learn programming" - - tools = [ - FunctionTool(fn=multiply), - FunctionTool(fn=add), - FunctionTool(fn=divide), - # FunctionTool.from_defaults(fn=search), - ] - llm_model_kwargs = { - "model": "llama3-70b-8192", # llama3 is not good with string formatting, llama3 8b is also bad at following instruction, 70b is better but still not as good as gpt-3.5-turbo - # mistral also not good: mixtral-8x7b-32768, but with better prompt, it can still work - "temperature": 0.0, - } - - gpt_3_5_turbo_model_kwargs = { - "model": "gpt-3.5-turbo", - } - - examples = [ - # r""" - # User: What is 9 - 3? - # You: { - # "thought": "I need to subtract 3 from 9, but there is no subtraction tool, so I ask llm_tool to answer the query.", - # "action": "llm_tool('What is 9 - 3?')" - # } - # """ - ] - # agent = ReActAgent( - # # examples=examples, - # tools=tools, - # max_steps=5, - # model_client=GroqAPIClient, - # model_kwargs=llm_model_kwargs, - # ) - # print(agent) - queries = [ - # "What is 2 times 3?", - # "What is 3 plus 4?", - "What is the capital of France? and what is 465 times 321 then add 95297 and then divide by 13.2?", - # "Li adapted her pet Apple in 2017 when Apple was only 2 months old, now we are at year 2024, how old is Li's pet Apple?", - "Give me 5 words rhyming with cool, and make a 4-sentence poem using them", - ] - """ - Results: mixtral-8x7b-32768, 0.9s per query - llama3-70b-8192, 1.8s per query - gpt-3.5-turbo, 2.2s per query - """ - import time - - generator = Generator( - model_client=GroqAPIClient(), - model_kwargs=llm_model_kwargs, - ) - # for i in range(3): - agent = ReActAgent( - tools=tools, - max_steps=5, - model_client=ModelClientType.GROQ(), - model_kwargs=llm_model_kwargs, - ) - # agent.llm_planner.print_prompt() - # print(agent) - - # vs not using agent - # print(agent.tools) - - average_time = 0 - for query in queries: - t0 = time.time() - answer = agent(query) - average_time += time.time() - t0 - answer_no_agent = generator(prompt_kwargs={"input_str": query}) - print(f"Answer with agent: {answer}") - print(f"Answer without agent: {answer_no_agent}") - print(f"Average time: {average_time / len(queries)}") diff --git a/lightrag/lightrag/components/data_process/data_components.py b/lightrag/lightrag/components/data_process/data_components.py index f712c052..f504e62e 100644 --- a/lightrag/lightrag/components/data_process/data_components.py +++ b/lightrag/lightrag/components/data_process/data_components.py @@ -33,7 +33,7 @@ def retriever_output_to_context_str( retriever_output: Union[RetrieverOutput, List[RetrieverOutput]], deduplicate: bool = False, ) -> str: - r"""The retrieved documents from one or mulitple queries. + r"""The retrieved documents from one or multiple queries. Deduplicate is especially helpful when you used query expansion. """ """ diff --git a/lightrag/lightrag/components/data_process/text_splitter.py b/lightrag/lightrag/components/data_process/text_splitter.py index f52f656a..38059902 100644 --- a/lightrag/lightrag/components/data_process/text_splitter.py +++ b/lightrag/lightrag/components/data_process/text_splitter.py @@ -346,4 +346,3 @@ def _extra_repr(self) -> str: return s -# test the execution llamaindex and langchain diff --git a/lightrag/lightrag/components/output_parsers/outputs.py b/lightrag/lightrag/components/output_parsers/outputs.py index 92ac9975..04cc3bb4 100644 --- a/lightrag/lightrag/components/output_parsers/outputs.py +++ b/lightrag/lightrag/components/output_parsers/outputs.py @@ -239,7 +239,6 @@ def __init__( self.output_processors = JsonParser() self.examples = examples - # TODO: make exclude works with both def format_instructions( self, format_type: Optional[DataClassFormatType] = None, diff --git a/lightrag/lightrag/core/base_data_class.py b/lightrag/lightrag/core/base_data_class.py index 6cbd33b2..c5d576ae 100644 --- a/lightrag/lightrag/core/base_data_class.py +++ b/lightrag/lightrag/core/base_data_class.py @@ -363,7 +363,9 @@ def to_schema(cls, exclude: ExcludeType = None) -> Dict[str, Dict[str, Any]]: excluded = deepcopy(exclude) else: excluded = None - return get_dataclass_schema(cls, excluded) + return get_dataclass_schema( + cls, excluded, getattr(cls, "__type_var_map__", None) + ) @classmethod def to_schema_str(cls, exclude: ExcludeType = None) -> str: @@ -476,6 +478,9 @@ def format_example_str( else: raise ValueError(f"Unsupported format type: {format_type}") + # TODO:support Generic[Type[T]] for the type of fields + # it will automatically use __type_var_map__ attribute + """Reserved for Agent to automatically create a dataclass and to manipulate the code""" diff --git a/lightrag/lightrag/core/default_prompt_template.py b/lightrag/lightrag/core/default_prompt_template.py index 6c8084b6..247a4d76 100644 --- a/lightrag/lightrag/core/default_prompt_template.py +++ b/lightrag/lightrag/core/default_prompt_template.py @@ -3,6 +3,12 @@ Use :ref:`Prompt ` class to manage it. """ +__all__ = [ + "LIGHTRAG_DEFAULT_PROMPT_ARGS", + "LIGHTRAG_DEFAULT_PROMPT_TRAINABLE_PARAMS", + "SIMPLE_DEFAULT_LIGHTRAG_SYSTEM_PROMPT", + "DEFAULT_LIGHTRAG_SYSTEM_PROMPT", +] # TODO: potentially make a data class for this LIGHTRAG_DEFAULT_PROMPT_ARGS = [ "task_desc_str", # task description diff --git a/lightrag/lightrag/core/functional.py b/lightrag/lightrag/core/functional.py index 83417816..7916ea3b 100644 --- a/lightrag/lightrag/core/functional.py +++ b/lightrag/lightrag/core/functional.py @@ -318,20 +318,36 @@ def is_dataclass_instance(obj): return hasattr(obj, "__dataclass_fields__") -def get_type_schema(type_obj, exclude: ExcludeType = None) -> str: +def get_type_schema( + type_obj, + exclude: ExcludeType = None, + type_var_map: Optional[Dict] = None, +) -> str: """Retrieve the type name, handling complex and nested types.""" origin = get_origin(type_obj) + type_var_map = type_var_map or {} + + # Replace type variables with their actual types to support Generic[T/To] + if hasattr(type_obj, "__origin__") and type_obj.__origin__ is not None: + type_obj = type_var_map.get(type_obj.__origin__, type_obj) + else: + type_obj = type_var_map.get(type_obj, type_obj) + if origin is Union: # Handle Optional[Type] and other unions args = get_args(type_obj) - types = [get_type_schema(arg, exclude) for arg in args if arg is not type(None)] + types = [ + get_type_schema(arg, exclude, type_var_map) + for arg in args + if arg is not type(None) + ] return ( f"Optional[{types[0]}]" if len(types) == 1 else f"Union[{', '.join(types)}]" ) elif origin in {List, list}: args = get_args(type_obj) if args: - inner_type = get_type_schema(args[0], exclude) + inner_type = get_type_schema(args[0], exclude, type_var_map) return f"List[{inner_type}]" else: return "List" @@ -339,34 +355,42 @@ def get_type_schema(type_obj, exclude: ExcludeType = None) -> str: elif origin in {Dict, dict}: args = get_args(type_obj) if args and len(args) >= 2: - key_type = get_type_schema(args[0], exclude) - value_type = get_type_schema(args[1], exclude) + key_type = get_type_schema(args[0], exclude, type_var_map) + value_type = get_type_schema(args[1], exclude, type_var_map) return f"Dict[{key_type}, {value_type}]" else: return "Dict" elif origin in {Set, set}: args = get_args(type_obj) - return f"Set[{get_type_schema(args[0], exclude)}]" if args else "Set" + return ( + f"Set[{get_type_schema(args[0],exclude, type_var_map)}]" if args else "Set" + ) elif origin is Sequence: args = get_args(type_obj) - return f"Sequence[{get_type_schema(args[0], exclude)}]" if args else "Sequence" + return ( + f"Sequence[{get_type_schema(args[0], exclude,type_var_map)}]" + if args + else "Sequence" + ) elif origin in {Tuple, tuple}: args = get_args(type_obj) if args: - return f"Tuple[{', '.join(get_type_schema(arg, exclude) for arg in args)}]" + return f"Tuple[{', '.join(get_type_schema(arg,exclude,type_var_map) for arg in args)}]" return "Tuple" elif is_dataclass(type_obj): # Recursively handle nested dataclasses - output = str(get_dataclass_schema(type_obj, exclude)) + output = str(get_dataclass_schema(type_obj, exclude, type_var_map)) return output return type_obj.__name__ if hasattr(type_obj, "__name__") else str(type_obj) def get_dataclass_schema( - cls, exclude: ExcludeType = None + cls, + exclude: ExcludeType = None, + type_var_map: Optional[Dict] = None, ) -> Dict[str, Dict[str, object]]: """Generate a schema dictionary for a dataclass including nested structures. @@ -374,11 +398,14 @@ def get_dataclass_schema( 2. Support nested dataclasses, even with generics like List, Dict, etc. 3. Support metadata in the dataclass fields. """ + if not is_dataclass(cls): raise ValueError( "Provided class is not a dataclass, please decorate your class with @dataclass" ) + type_var_map = type_var_map or {} + schema = {"type": cls.__name__, "properties": {}, "required": []} # get the exclude list for the current class current_exclude = exclude.get(cls.__name__, []) if exclude else [] @@ -386,7 +413,9 @@ def get_dataclass_schema( if f.name in current_exclude: continue # prepare field schema, it weill be done recursively for nested dataclasses - field_schema = {"type": get_type_schema(f.type, exclude)} + + field_type = type_var_map.get(f.type, f.type) + field_schema = {"type": get_type_schema(field_type, exclude, type_var_map)} # check required field is_required = _is_required_field(f) diff --git a/lightrag/lightrag/core/types.py b/lightrag/lightrag/core/types.py index efcea9a7..deac261b 100644 --- a/lightrag/lightrag/core/types.py +++ b/lightrag/lightrag/core/types.py @@ -13,6 +13,7 @@ Literal, Callable, Awaitable, + Type, ) from collections import OrderedDict from dataclasses import ( @@ -43,6 +44,7 @@ logger = logging.getLogger(__name__) T_co = TypeVar("T_co", covariant=True) +T = TypeVar("T") # invariant type ####################################################################################### @@ -318,6 +320,13 @@ def add(a, b): ) +_action_desc = """FuncName() \ +Valid function call expression. \ +Example: "FuncName(a=1, b=2)" \ +Follow the data type specified in the function parameters.\ +e.g. for Type object with x,y properties, use "ObjectType(x=1, y=2)""" + + @dataclass class FunctionExpression(DataClass): __doc__ = r"""The data modeling of a function expression for a call, including the name and arguments. @@ -357,13 +366,7 @@ def add(a, b): action: str = field( default_factory=required_field, # metadata={"desc": "FuncName(, )"}, - metadata={ - "desc": """FuncName() \ - Valid function call expression. \ - Example: "FuncName(a=1, b=2)" \ - Follow the data type specified in the function parameters.\ - e.g. for Type object with x,y properties, use "ObjectType(x=1, y=2)""" - }, + metadata={"desc": _action_desc}, ) @classmethod @@ -429,34 +432,57 @@ class FunctionOutput(DataClass): # Data modeling for agent component ###################################################################################### @dataclass -class StepOutput(DataClass): - __doc__ = r"""The output of a single step in the agent.""" +class StepOutput(DataClass, Generic[T]): + __doc__ = r"""The output of a single step in the agent. Suits for serial planning agent such as React""" step: int = field( default=0, metadata={"desc": "The order of the step in the agent"} ) - thought: Optional[str] = field( - default="", metadata={"desc": "The thought of the agent in the step"} - ) - action: str = field( - default="", metadata={"desc": "The action of the agent in the step"} - ) - fun_name: Optional[str] = field( - default=None, metadata={"desc": "The function named parsed from action"} - ) - fun_args: Optional[List[Any]] = field( - default=None, - metadata={"desc": "The function positional arguments parsed from action"}, + + # This action can be in Function, or Function Exptression, or just str + # it includes the thought and action already + # directly the output from planner LLMs + action: T = field( + default=None, metadata={"desc": "The action the agent takes at this step"} ) - fun_kwargs: Optional[Dict[str, Any]] = field( - default=None, - metadata={"desc": "The function keyword arguments parsed from action"}, + + function: Optional[Function] = field( + default=None, metadata={"desc": "The parsed function from the action"} ) + observation: Optional[str] = field( - default=None, metadata={"desc": "The result of the action"} + default=None, metadata={"desc": "The execution result shown for this action"} ) - def __str__(self): - return f"Thought {self.step}: {self.thought}\nAction {self.step}: {self.action}\nObservation {self.step}: {self.observation}" + @classmethod + def with_action_type(cls, action_type: Type[T]) -> Type["StepOutput[T]"]: + """ + Create a new StepOutput class with the specified action type. + + Use this if you want to create schema for StepOutput with a specific action type. + + Args: + action_type (Type[T]): The type to set for the action attribute. + + Returns: + Type[StepOutput[T]]: A new subclass of StepOutput with the specified action type. + + Example: + + .. code-block:: python + + from lightrag.core.types import StepOutput, FunctionExpression + + StepOutputWithFunctionExpression = StepOutput.with_action_type(FunctionExpression) + """ + # Create a new type variable map + type_var_map = {T: action_type} + + # Create a new subclass with the updated type + new_cls = type(cls.__name__, (cls,), {"__type_var_map__": type_var_map}) + + # Update the __annotations__ to reflect the new type of action + new_cls.__annotations__["action"] = action_type + return new_cls ####################################################################################### diff --git a/lightrag/poetry.lock b/lightrag/poetry.lock index 02d8f4b7..5a453e03 100644 --- a/lightrag/poetry.lock +++ b/lightrag/poetry.lock @@ -750,20 +750,20 @@ files = [ [[package]] name = "networkx" -version = "3.3" +version = "3.2.1" description = "Python package for creating and manipulating graphs and networks" optional = false -python-versions = ">=3.10" +python-versions = ">=3.9" files = [ - {file = "networkx-3.3-py3-none-any.whl", hash = "sha256:28575580c6ebdaf4505b22c6256a2b9de86b316dc63ba9e93abde3d78dfdbcf2"}, - {file = "networkx-3.3.tar.gz", hash = "sha256:0c127d8b2f4865f59ae9cb8aafcd60b5c70f3241ebd66f7defad7c4ab90126c9"}, + {file = "networkx-3.2.1-py3-none-any.whl", hash = "sha256:f18c69adc97877c42332c170849c96cefa91881c99a7cb3e95b7c659ebdc1ec2"}, + {file = "networkx-3.2.1.tar.gz", hash = "sha256:9f1bb5cf3409bf324e0a722c20bdb4c20ee39bf1c30ce8ae499c8502b0b5e0c6"}, ] [package.extras] -default = ["matplotlib (>=3.6)", "numpy (>=1.23)", "pandas (>=1.4)", "scipy (>=1.9,!=1.11.0,!=1.11.1)"] -developer = ["changelist (==0.5)", "mypy (>=1.1)", "pre-commit (>=3.2)", "rtoml"] -doc = ["myst-nb (>=1.0)", "numpydoc (>=1.7)", "pillow (>=9.4)", "pydata-sphinx-theme (>=0.14)", "sphinx (>=7)", "sphinx-gallery (>=0.14)", "texext (>=0.6.7)"] -extra = ["lxml (>=4.6)", "pydot (>=2.0)", "pygraphviz (>=1.12)", "sympy (>=1.10)"] +default = ["matplotlib (>=3.5)", "numpy (>=1.22)", "pandas (>=1.4)", "scipy (>=1.9,!=1.11.0,!=1.11.1)"] +developer = ["changelist (==0.4)", "mypy (>=1.1)", "pre-commit (>=3.2)", "rtoml"] +doc = ["nb2plots (>=0.7)", "nbconvert (<7.9)", "numpydoc (>=1.6)", "pillow (>=9.4)", "pydata-sphinx-theme (>=0.14)", "sphinx (>=7)", "sphinx-gallery (>=0.14)", "texext (>=0.6.7)"] +extra = ["lxml (>=4.6)", "pydot (>=1.4.2)", "pygraphviz (>=1.11)", "sympy (>=1.10)"] test = ["pytest (>=7.2)", "pytest-cov (>=4.0)"] [[package]] @@ -966,13 +966,13 @@ files = [ [[package]] name = "openai" -version = "1.35.12" +version = "1.35.13" description = "The official Python library for the openai API" optional = false python-versions = ">=3.7.1" files = [ - {file = "openai-1.35.12-py3-none-any.whl", hash = "sha256:c3a8c9be524480ae32a3212b78d90786b112844eb8f96235ebb8bc5f35cb1e72"}, - {file = "openai-1.35.12.tar.gz", hash = "sha256:2dd0f1d9ff34bf6bc89d15246245369d281b2ba01ae07eae5555a00da4b51e0b"}, + {file = "openai-1.35.13-py3-none-any.whl", hash = "sha256:36ec3e93e0d1f243f69be85c89b9221a471c3e450dfd9df16c9829e3cdf63e60"}, + {file = "openai-1.35.13.tar.gz", hash = "sha256:c684f3945608baf7d2dcc0ef3ee6f3e27e4c66f21076df0b47be45d57e6ae6e4"}, ] [package.dependencies] @@ -2048,5 +2048,5 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess [metadata] lock-version = "2.0" -python-versions = ">=3.10, <4.0" -content-hash = "b4026b7c4601d9e9ce0cda2ed24add4689fa54ac6c990a2dcdc7095733206aed" +python-versions = ">=3.9, <4.0" +content-hash = "933e0a902cffe3e2808a2337d0eeb6a2b566998c7bfb5e3b2edd8e289566cd39" diff --git a/lightrag/pyproject.toml b/lightrag/pyproject.toml index 44d6a9de..de58e59d 100644 --- a/lightrag/pyproject.toml +++ b/lightrag/pyproject.toml @@ -1,8 +1,8 @@ [tool.poetry] name = "lightrag" -version = "0.0.0-alpha.16" -description = "The 'PyTorch' library for LLM applications." +version = "0.0.0-beta.1" +description = "The Lightning Library for LLM Applications." authors = ["Li Yin "] readme = "README.md" repository = "https://github.com/SylphAI-Inc/LightRAG" @@ -32,7 +32,7 @@ packages = [{ include = "lightrag", from = "." }] [tool.poetry.dependencies] -python = ">=3.10, <4.0" +python = ">=3.9, <4.0" python-dotenv = "^1.0.1" backoff = "^2.2.1" diff --git a/lightrag/test.py b/lightrag/test.py new file mode 100644 index 00000000..0a1ef43a --- /dev/null +++ b/lightrag/test.py @@ -0,0 +1,19 @@ +from lightrag.core.types import StepOutput, FunctionExpression +from dataclasses import dataclass + + +@dataclass +class MyStepOutput(StepOutput[FunctionExpression]): + pass + + +StepOutputClass = StepOutput.with_action_type(FunctionExpression) +schema = StepOutputClass.to_schema() + +print(schema) + +instance = StepOutput( + step=0, action=FunctionExpression(thought="hello", action="print") +) + +print(instance) diff --git a/lightrag/tests/test_lazy_import.py b/lightrag/tests/test_lazy_import.py new file mode 100644 index 00000000..90dc4d14 --- /dev/null +++ b/lightrag/tests/test_lazy_import.py @@ -0,0 +1,32 @@ +from types import ModuleType +import pytest +import unittest +from lightrag.utils.lazy_import import safe_import + + +class TestSafeImport(unittest.TestCase): + def test_import_installed_package(self): + module = safe_import("numpy", "Please install math with: pip install numpy") + self.assertIsInstance( + module, ModuleType, f"Expected module type, got {type(module)}" + ) + self.assertTrue( + hasattr(module, "__version__"), + "Module 'numpy' should have attribute '__version__'", + ) + + def test_import_nonexistent_package(self): + with self.assertRaises(ImportError) as cm: + safe_import( + "nonexistent_package", + "Please install nonexistent_package with: pip install nonexistent_package", + ) + self.assertIn( + "Please install nonexistent_package with: pip install nonexistent_package", + str(cm.exception), + (f"Expected error message not found in {str(cm.exception)}"), + ) + + +if __name__ == "__main__": + pytest.main() diff --git a/lightrag/tests/test_model_client.py b/lightrag/tests/test_model_client.py new file mode 100644 index 00000000..5b21760b --- /dev/null +++ b/lightrag/tests/test_model_client.py @@ -0,0 +1,40 @@ +from typing import Any +import unittest +from unittest.mock import patch + +from lightrag.components.model_client import OpenAIClient as OpenAIClientLazyImport + + +class TestLazyImportSubclassing(unittest.TestCase): + def test_subclassing_raises_error(self): + with self.assertRaises(TypeError): + + class InvalidCustomizeOpenAIClient(OpenAIClientLazyImport): + def __init__(self): + super().__init__() + + def parse_chat_completion(self, completion: Any) -> Any: + """Parse the completion to a str.""" + print(f"completion: {completion}") + return self.chat_completion_parser(completion) + + InvalidCustomizeOpenAIClient() + + @patch.dict("os.environ", {"OPENAI_API_KEY": "test_api_key"}) + def test_correct_subclassing(self): + from lightrag.components.model_client.openai_client import OpenAIClient + + class CorrectCustomizeOpenAIClient(OpenAIClient): + def __init__(self): + super().__init__() + + def parse_chat_completion(self, completion: Any) -> Any: + """Parse the completion to a str.""" + print(f"completion: {completion}") + return self.chat_completion_parser(completion) + + CorrectCustomizeOpenAIClient() + + +if __name__ == "__main__": + unittest.main() diff --git a/lightrag/tests/test_text_splitter.py b/lightrag/tests/test_text_splitter.py index 319fd5d7..0f686646 100644 --- a/lightrag/tests/test_text_splitter.py +++ b/lightrag/tests/test_text_splitter.py @@ -66,4 +66,4 @@ def test_empty_text_handling(self): if __name__ == "__main__": - unittest.main() + unittest.main() \ No newline at end of file diff --git a/use_cases/yaml_output.py b/use_cases/yaml_output.py index 0a26367c..7463028d 100644 --- a/use_cases/yaml_output.py +++ b/use_cases/yaml_output.py @@ -3,6 +3,7 @@ from lightrag.components.model_client import GroqAPIClient from lightrag.components.output_parsers import YamlOutputParser from dataclasses import dataclass +from lightrag.utils import setup_env from lightrag.core.base_data_class import DataClass, field @@ -20,17 +21,18 @@ class JokeOutput(DataClass): punchline="Because he was outstanding in his field.", ) +setup_env() + +# Our parser not only helps format the output format, but also examples class JokeGenerator(Component): def __init__(self): super().__init__() - yaml_parser = YamlOutputParser(data_class=JokeOutput, example=joke_example) + yaml_parser = YamlOutputParser(data_class=JokeOutput, examples=[joke_example]) self.generator = Generator( model_client=GroqAPIClient(), model_kwargs={"model": "llama3-8b-8192", "temperature": 1.0}, - preset_prompt_kwargs={ - "output_format_str": yaml_parser.format_instructions() - }, + prompt_kwargs={"output_format_str": yaml_parser.format_instructions()}, output_processors=yaml_parser, ) @@ -54,3 +56,102 @@ def call(self, query: str, model_kwargs: dict = {}) -> JokeOutput: answer = joke_generator.call("Tell me two jokes.", model_kwargs={"temperature": 1}) print(answer) print(f"typeof answer: {type(answer)}") + + from langchain_core.output_parsers import ( + JsonOutputParser as LangchainJsonOutputParser, + ) + from langchain_core.pydantic_v1 import BaseModel, Field + from typing import TypeVar, Generic + + T = TypeVar("T") + # 1. we dont include default value in the schema as it is how the program wants to handle it and not the job of llm + # 2. nested class is defined in definitions. + # we directly include in that type, and we should remove the additional "type" and make that part a dict instead of a string. + ours = { + "type": "MyStepOutput", + "properties": { + "step": {"type": "int", "desc": "The order of the step in the agent"}, + "action": { + "type": "{'type': 'FunctionExpression', 'properties': {'thought': {'type': 'Optional[str]', 'desc': 'Why the function is called'}, 'action': {'type': 'str', 'desc': 'FuncName() Valid function call expression. Example: \"FuncName(a=1, b=2)\" Follow the data type specified in the function parameters.e.g. for Type object with x,y properties, use \"ObjectType(x=1, y=2)'}}, 'required': ['action']}", + "desc": "The action the agent takes at this step", + }, + "function": { + "type": "Optional[{'type': 'Function', 'properties': {'thought': {'type': 'Optional[str]', 'desc': 'Why the function is called'}, 'name': {'type': 'str', 'desc': 'The name of the function'}, 'args': {'type': 'Optional[List[object]]', 'desc': 'The positional arguments of the function'}, 'kwargs': {'type': 'Optional[Dict[str, object]]', 'desc': 'The keyword arguments of the function'}}, 'required': []}]", + "desc": "The parsed function from the action", + }, + "observation": { + "type": "Optional[str]", + "desc": "The execution result shown for this action", + }, + }, + "required": [], + } + langchains = { + "properties": { + "setup": { + "title": "Setup", + "description": "question to set up a joke", + "default": "", + "allOf": [{"$ref": "#/definitions/JokeOutput"}], + }, + "punchline": { + "title": "Punchline", + "description": "answer to resolve the joke", + "default": "", + "type": "string", + }, + }, + "definitions": { + "JokeOutput": { + "title": "JokeOutput", + "type": "object", + "properties": { + "setup": { + "title": "Setup", + "default": "", + "desc": "question to set up a joke", + "type": "string", + }, + "punchline": { + "title": "Punchline", + "default": "", + "desc": "answer to resolve the joke", + "type": "string", + }, + }, + } + }, + } + + # subtype: allOf": [{"$ref": "#/definitions/JokeOutput"}]}, + definitions = { + "definitions": { + "JokeOutput": { + "title": "JokeOutput", + "type": "object", + "properties": { + "setup": { + "title": "Setup", + "default": "", + "desc": "question to set up a joke", + "type": "string", + }, + "punchline": { + "title": "Punchline", + "default": "", + "desc": "answer to resolve the joke", + "type": "string", + }, + }, + } + } + } + + class JokeOutputV1(BaseModel, Generic[T]): + setup: JokeOutput = Field(description="question to set up a joke", default="") + punchline: str = Field(description="answer to resolve the joke", default="") + + json_parser = LangchainJsonOutputParser(pydantic_object=JokeOutputV1) + print("langchain instruction") + # failed to detect the generic type of the pydantic object + print(json_parser.get_format_instructions())