From 6cb9a5cbeaabaedee10f2ad689a778c8dfb8088a Mon Sep 17 00:00:00 2001 From: Wenzhe Xue Date: Fri, 9 Feb 2024 11:21:05 -0800 Subject: [PATCH 01/12] wip, init commit --- log10/feedback/feedback.py | 65 +++++++++++++++++++++++++++++++++ log10/feedback/feedback_task.py | 55 ++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 log10/feedback/feedback.py create mode 100644 log10/feedback/feedback_task.py diff --git a/log10/feedback/feedback.py b/log10/feedback/feedback.py new file mode 100644 index 00000000..87458656 --- /dev/null +++ b/log10/feedback/feedback.py @@ -0,0 +1,65 @@ +import click +import logging + +import httpx +from dotenv import load_dotenv + +from log10.llm import Log10Config + + +# def create(name: str, task_schema: dict) -> httpx.Response: +# """ +# Example: +# >>> from log10.feedback import feedback, feedback_task +# >>> task = feedback_task.create(name="summarization", task_schema={...}) +# >>> task_id = task.id +# >>> fb = feedback.create(task_id=task_id, rate={...}) +# """ + +load_dotenv() + +logging.basicConfig( + format="[%(asctime)s - %(name)s - %(levelname)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", +) +logger: logging.Logger = logging.getLogger("LOG10") +logger.setLevel(logging.INFO) + +class Feedback: + feedback_create_url = "/api/v1/feedback" + + def __init__(self, log10_config: Log10Config = None): + self._log10_config = log10_config or Log10Config() + self._http_client = httpx.Client() + + def _post_request(self, url: str, json_payload: dict) -> httpx.Response: + headers = {"x-log10-token": self._log10_config.token, "Content-Type": "application/json"} + json_payload["organization_id"] = self._log10_config.org_id + try: + res = self._http_client.post( + self._log10_config.url + url, headers=headers, json=json_payload + ) + res.raise_for_status() + return res + except Exception as e: + logger.error(e) + raise + + def create(self, task_id: str, rate: dict) -> httpx.Response: + """ + Example: + >>> from log10.feedback import Feedback + >>> fb = Feedback() + >>> fb.create(task_id="task_id", rate={...}) + """ + json_payload = {"task_id": task_id, "rate": rate} + res = self._post_request(self.feedback_create_url, json_payload) + return res + +@click.command() +@click.option("--task_id", help="Task ID") +@click.option("--rate", help="Rate in JSON format") +def create_feedback(task_id, rate): + fb = Feedback() + feedback = fb.create(task_id=task_id, rate=rate) + print(feedback) diff --git a/log10/feedback/feedback_task.py b/log10/feedback/feedback_task.py new file mode 100644 index 00000000..41eed5c0 --- /dev/null +++ b/log10/feedback/feedback_task.py @@ -0,0 +1,55 @@ +import click +import logging + +import httpx +from dotenv import load_dotenv + +from log10.llm import Log10Config + + +load_dotenv() + +logging.basicConfig( + format="[%(asctime)s - %(name)s - %(levelname)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", +) +logger: logging.Logger = logging.getLogger("LOG10") +logger.setLevel(logging.INFO) +class FeedbackTask: + feedback_task_create_url = "/api/v1/feedback_task" + def __init__(self, log10_config: Log10Config = None): + self._log10_config = log10_config or Log10Config() + self._http_client = httpx.Client() + + def _post_request(self, url: str, json_payload: dict) -> httpx.Response: + headers = {"x-log10-token": self._log10_config.token, "Content-Type": "application/json"} + json_payload["organization_id"] = self._log10_config.org_id + try: + res = self._http_client.post( + self._log10_config.url + url, headers=headers, json=json_payload + ) + res.raise_for_status() + return res + except Exception as e: + logger.error(e) + raise + + def create(self, name: str, task_schema: dict) -> httpx.Response: + """ + Example: + >>> from log10.feedback.feedback_task import FeedbackTask + >>> feedback_task = FeedbackTask() + >>> task = feedback_task.create(name="summarization", task_schema={...}) + """ + json_payload = {"name": name, "task_schema": task_schema} + res = self._post_request(self.feedback_task_create_url, json_payload) + return res + +# create a cli interface for FeebackTask.create function +@click.command() +@click.option("--name", help="Name of the task") +@click.option("--task_schema", help="Task schema") +def create_feedback_task(name, task_schema): + fb_task = FeedbackTask() + task = fb_task.create(name=name, task_schema=task_schema) + print(task) From df7a078954507507511263bafe8a4fe113211076 Mon Sep 17 00:00:00 2001 From: Wenzhe Xue Date: Fri, 9 Feb 2024 15:20:41 -0800 Subject: [PATCH 02/12] wip add cli log10 feedback or log10 feedback-task create --- log10/__main__.py | 32 ++++++++++++++++++++++++++++++++ log10/feedback/feedback.py | 15 +++++++++------ log10/feedback/feedback_task.py | 15 +++++++++------ pyproject.toml | 3 +++ 4 files changed, 53 insertions(+), 12 deletions(-) create mode 100644 log10/__main__.py diff --git a/log10/__main__.py b/log10/__main__.py new file mode 100644 index 00000000..52cf4a1c --- /dev/null +++ b/log10/__main__.py @@ -0,0 +1,32 @@ +import click + +from log10.feedback.feedback import create_feedback +from log10.feedback.feedback_task import create_feedback_task + + +@click.group() +def cli(): + pass + + +@click.group() +def feedback(): + pass + + +@click.group() +def feedback_task(): + pass + + +cli.add_command(feedback) +feedback.add_command(create_feedback, "create") +cli.add_command(feedback_task) +feedback_task.add_command(create_feedback_task, "create") + +if __name__ == "__main__": + cli() + + +# cli.add_command(feedback.create_feedback) +# cli.add_command(feedback_task.create_feedback_task) diff --git a/log10/feedback/feedback.py b/log10/feedback/feedback.py index 87458656..70e8844c 100644 --- a/log10/feedback/feedback.py +++ b/log10/feedback/feedback.py @@ -1,6 +1,6 @@ -import click import logging +import click import httpx from dotenv import load_dotenv @@ -57,9 +57,12 @@ def create(self, task_id: str, rate: dict) -> httpx.Response: return res @click.command() -@click.option("--task_id", help="Task ID") -@click.option("--rate", help="Rate in JSON format") +@click.option("--task_id", prompt="Enter task id", help="Task ID") +@click.option("--rate", prompt="Enter task rate", help="Rate in JSON format") def create_feedback(task_id, rate): - fb = Feedback() - feedback = fb.create(task_id=task_id, rate=rate) - print(feedback) + click.echo("Creating feedback") + click.echo(f"Task ID: {task_id}") + click.echo(f"Rate: {rate}") + # fb = Feedback() + # feedback = fb.create(task_id=task_id, rate=rate) + # click.echo(feedback) diff --git a/log10/feedback/feedback_task.py b/log10/feedback/feedback_task.py index 41eed5c0..d81a5996 100644 --- a/log10/feedback/feedback_task.py +++ b/log10/feedback/feedback_task.py @@ -1,6 +1,6 @@ -import click import logging +import click import httpx from dotenv import load_dotenv @@ -47,9 +47,12 @@ def create(self, name: str, task_schema: dict) -> httpx.Response: # create a cli interface for FeebackTask.create function @click.command() -@click.option("--name", help="Name of the task") -@click.option("--task_schema", help="Task schema") +@click.option("--name", prompt="Enter feedback task name", help="Name of the task") +@click.option("--task_schema", prompt="Enter feedback task schema", help="Task schema") def create_feedback_task(name, task_schema): - fb_task = FeedbackTask() - task = fb_task.create(name=name, task_schema=task_schema) - print(task) + click.echo("Creating feedback task") + click.echo(f"Name: {name}") + click.echo(f"Task Schema: {task_schema}") + # fb_task = FeedbackTask() + # task = fb_task.create(name=name, task_schema=task_schema) + # click.echo(task) diff --git a/pyproject.toml b/pyproject.toml index c953fe35..b7bd4f85 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,9 @@ packages = [ { include = "log10" }, ] +[tool.poetry.scripts] +log10 = "log10.__main__:cli" + [tool.poetry.group.dev.dependencies] build = "^0.10.0" pytest = "^8.0.0" From c42a2e55b72b8918aa291acb7816f9514facb298 Mon Sep 17 00:00:00 2001 From: Wenzhe Xue Date: Fri, 9 Feb 2024 15:53:28 -0800 Subject: [PATCH 03/12] format --- log10/feedback/feedback.py | 6 +++--- log10/feedback/feedback_task.py | 8 +++++--- pyproject.toml | 8 ++++---- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/log10/feedback/feedback.py b/log10/feedback/feedback.py index 70e8844c..7ac12b5d 100644 --- a/log10/feedback/feedback.py +++ b/log10/feedback/feedback.py @@ -25,6 +25,7 @@ logger: logging.Logger = logging.getLogger("LOG10") logger.setLevel(logging.INFO) + class Feedback: feedback_create_url = "/api/v1/feedback" @@ -36,9 +37,7 @@ def _post_request(self, url: str, json_payload: dict) -> httpx.Response: headers = {"x-log10-token": self._log10_config.token, "Content-Type": "application/json"} json_payload["organization_id"] = self._log10_config.org_id try: - res = self._http_client.post( - self._log10_config.url + url, headers=headers, json=json_payload - ) + res = self._http_client.post(self._log10_config.url + url, headers=headers, json=json_payload) res.raise_for_status() return res except Exception as e: @@ -56,6 +55,7 @@ def create(self, task_id: str, rate: dict) -> httpx.Response: res = self._post_request(self.feedback_create_url, json_payload) return res + @click.command() @click.option("--task_id", prompt="Enter task id", help="Task ID") @click.option("--rate", prompt="Enter task rate", help="Rate in JSON format") diff --git a/log10/feedback/feedback_task.py b/log10/feedback/feedback_task.py index d81a5996..15769dcf 100644 --- a/log10/feedback/feedback_task.py +++ b/log10/feedback/feedback_task.py @@ -15,8 +15,11 @@ ) logger: logging.Logger = logging.getLogger("LOG10") logger.setLevel(logging.INFO) + + class FeedbackTask: feedback_task_create_url = "/api/v1/feedback_task" + def __init__(self, log10_config: Log10Config = None): self._log10_config = log10_config or Log10Config() self._http_client = httpx.Client() @@ -25,9 +28,7 @@ def _post_request(self, url: str, json_payload: dict) -> httpx.Response: headers = {"x-log10-token": self._log10_config.token, "Content-Type": "application/json"} json_payload["organization_id"] = self._log10_config.org_id try: - res = self._http_client.post( - self._log10_config.url + url, headers=headers, json=json_payload - ) + res = self._http_client.post(self._log10_config.url + url, headers=headers, json=json_payload) res.raise_for_status() return res except Exception as e: @@ -45,6 +46,7 @@ def create(self, name: str, task_schema: dict) -> httpx.Response: res = self._post_request(self.feedback_task_create_url, json_payload) return res + # create a cli interface for FeebackTask.create function @click.command() @click.option("--name", prompt="Enter feedback task name", help="Name of the task") diff --git a/pyproject.toml b/pyproject.toml index b7bd4f85..9c7dd586 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,17 +51,17 @@ together = "^0.2.7" [tool.ruff] # Never enforce `E501` (line length violations). -ignore = ["C901", "E501", "E741", "F402", "F823" ] -select = ["C", "E", "F", "I", "W"] +lint.ignore = ["C901", "E501", "E741", "F402", "F823" ] +lint.select = ["C", "E", "F", "I", "W"] line-length = 119 # Ignore import violations in all `__init__.py` files. -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] "__init__.py" = ["E402", "F401", "F403", "F811"] "log10/langchain.py" = ["E402"] "examples/logging/*.py" = ["E402", "F821"] -[tool.ruff.isort] +[tool.ruff.lint.isort] lines-after-imports = 2 known-first-party = ["log10"] From 39e4cc626e4503cdee267683477fca9d57be63c1 Mon Sep 17 00:00:00 2001 From: Wenzhe Xue Date: Mon, 12 Feb 2024 11:49:48 -0800 Subject: [PATCH 04/12] wip stash changes --- examples/feedback/create_feedback_task.py | 20 ++++++++++++ examples/feedback/simple_feedback.py | 38 +++++++++++++++++++++++ log10/__main__.py | 4 --- log10/feedback/feedback.py | 2 +- log10/feedback/feedback_task.py | 17 +++++++--- 5 files changed, 72 insertions(+), 9 deletions(-) create mode 100644 examples/feedback/create_feedback_task.py create mode 100644 examples/feedback/simple_feedback.py diff --git a/examples/feedback/create_feedback_task.py b/examples/feedback/create_feedback_task.py new file mode 100644 index 00000000..35b8c3ba --- /dev/null +++ b/examples/feedback/create_feedback_task.py @@ -0,0 +1,20 @@ +import logging +from log10.feedback.feedback_task import FeedbackTask + +httpx_logger = logging.getLogger("httpx") +httpx_logger.setLevel(logging.DEBUG) + +feedback_task = FeedbackTask() +t_s = { + "type": "object", + "properties": { + "feedback": { + "type": "string", + "enum": ["😀", "😬", "😐", "🙁", "😫"] + } + } +} +# convert t_s to json +import json +t_s = json.dumps(t_s) +task = feedback_task.create(name="emo", task_schema=t_s) diff --git a/examples/feedback/simple_feedback.py b/examples/feedback/simple_feedback.py new file mode 100644 index 00000000..9a57d43e --- /dev/null +++ b/examples/feedback/simple_feedback.py @@ -0,0 +1,38 @@ +from pprint import pprint +from typing import Literal + +from pydantic import BaseModel, Field + +from log10.feedback.feedback import Feedback +from log10.feedback.feedback_task import FeedbackTask +from log10.load import OpenAI + + +class EmojiFeedback(BaseModel): + feedback: Literal["😀", "🙁"] = Field(..., description="User feedback with emojis") + + +client = OpenAI() +completion = client.chat.completions.create( + model="gpt-3.5-turbo", + messages=[ + { + "role": "system", + "content": "You are the most knowledgable Star Wars guru on the planet", + }, + { + "role": "user", + "content": "Write the time period of all the Star Wars movies and spinoffs?", + }, + ], +) +completion_id = completion.choices[0].id + +task = FeedbackTask() +res = task.create(name="emoji_feedback_task", task_schema=EmojiFeedback.model_json_schema()) +task_id = res["id"] +pprint(task) +# Example usage +fb = Feedback() +res = fb.create(task_id=task_id, data=EmojiFeedback(feedback="😀").model_dump_json(), completion_tags_selector=[str(completion_id)]) +pprint(res) diff --git a/log10/__main__.py b/log10/__main__.py index 52cf4a1c..ca446d4e 100644 --- a/log10/__main__.py +++ b/log10/__main__.py @@ -26,7 +26,3 @@ def feedback_task(): if __name__ == "__main__": cli() - - -# cli.add_command(feedback.create_feedback) -# cli.add_command(feedback_task.create_feedback_task) diff --git a/log10/feedback/feedback.py b/log10/feedback/feedback.py index 7ac12b5d..e021151a 100644 --- a/log10/feedback/feedback.py +++ b/log10/feedback/feedback.py @@ -34,7 +34,7 @@ def __init__(self, log10_config: Log10Config = None): self._http_client = httpx.Client() def _post_request(self, url: str, json_payload: dict) -> httpx.Response: - headers = {"x-log10-token": self._log10_config.token, "Content-Type": "application/json"} + headers = {"x-log10-token": self._log10_config.token, "x-log10-organization": self._log10_config.org_id, "Content-Type": "application/json"} json_payload["organization_id"] = self._log10_config.org_id try: res = self._http_client.post(self._log10_config.url + url, headers=headers, json=json_payload) diff --git a/log10/feedback/feedback_task.py b/log10/feedback/feedback_task.py index 15769dcf..d2319d5b 100644 --- a/log10/feedback/feedback_task.py +++ b/log10/feedback/feedback_task.py @@ -17,17 +17,21 @@ logger.setLevel(logging.INFO) + class FeedbackTask: - feedback_task_create_url = "/api/v1/feedback_task" + feedback_task_create_url = "api/v1/feedback_task" def __init__(self, log10_config: Log10Config = None): self._log10_config = log10_config or Log10Config() self._http_client = httpx.Client() def _post_request(self, url: str, json_payload: dict) -> httpx.Response: - headers = {"x-log10-token": self._log10_config.token, "Content-Type": "application/json"} + headers = {"x-log10-token": self._log10_config.token, "Content-Type": "application/json", "x-log10-organization": self._log10_config.org_id} json_payload["organization_id"] = self._log10_config.org_id try: + from pprint import pprint + pprint(f"{headers=}") + pprint(f"{json_payload=}") res = self._http_client.post(self._log10_config.url + url, headers=headers, json=json_payload) res.raise_for_status() return res @@ -35,14 +39,19 @@ def _post_request(self, url: str, json_payload: dict) -> httpx.Response: logger.error(e) raise - def create(self, name: str, task_schema: dict) -> httpx.Response: + def create(self, task_schema: dict, name: str = None, instruction: str = None) -> httpx.Response: """ Example: >>> from log10.feedback.feedback_task import FeedbackTask >>> feedback_task = FeedbackTask() >>> task = feedback_task.create(name="summarization", task_schema={...}) """ - json_payload = {"name": name, "task_schema": task_schema} + json_payload = {"task_schema": task_schema} + if name: + json_payload["name"] = name + if instruction: + json_payload["instruction"] = instruction + res = self._post_request(self.feedback_task_create_url, json_payload) return res From 173e1ad6ca1f8140b811469c5b868e960a94d2f4 Mon Sep 17 00:00:00 2001 From: Wenzhe Xue Date: Mon, 12 Feb 2024 12:25:46 -0800 Subject: [PATCH 05/12] made example working to create task and post feedback w/ api --- examples/feedback/create_feedback_task.py | 18 +++++++++++++++--- log10/feedback/feedback.py | 6 +++--- log10/feedback/feedback_task.py | 5 +---- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/examples/feedback/create_feedback_task.py b/examples/feedback/create_feedback_task.py index 35b8c3ba..967ac524 100644 --- a/examples/feedback/create_feedback_task.py +++ b/examples/feedback/create_feedback_task.py @@ -1,5 +1,8 @@ import logging +from pydantic import BaseModel, Field +from typing import Literal from log10.feedback.feedback_task import FeedbackTask +from log10.feedback.feedback import Feedback httpx_logger = logging.getLogger("httpx") httpx_logger.setLevel(logging.DEBUG) @@ -14,7 +17,16 @@ } } } +class EmojiFeedback(BaseModel): + feedback: Literal["😀", "🙁"] = Field(..., description="User feedback with emojis") + +eft = EmojiFeedback(feedback="😀") + # convert t_s to json -import json -t_s = json.dumps(t_s) -task = feedback_task.create(name="emo", task_schema=t_s) +task = feedback_task.create(name="emo", task_schema=eft.model_json_schema()) +task_dump = task.json() +print(task_dump["id"]) + +fb = Feedback() +print(eft.model_dump_json()) +fb.create(task_id=task_dump["id"], rate=eft.model_dump(), completion_tags_selector=["give_me_feedback"]) \ No newline at end of file diff --git a/log10/feedback/feedback.py b/log10/feedback/feedback.py index e021151a..df95a8a5 100644 --- a/log10/feedback/feedback.py +++ b/log10/feedback/feedback.py @@ -27,7 +27,7 @@ class Feedback: - feedback_create_url = "/api/v1/feedback" + feedback_create_url = "api/v1/feedback" def __init__(self, log10_config: Log10Config = None): self._log10_config = log10_config or Log10Config() @@ -44,14 +44,14 @@ def _post_request(self, url: str, json_payload: dict) -> httpx.Response: logger.error(e) raise - def create(self, task_id: str, rate: dict) -> httpx.Response: + def create(self, task_id: str, rate: dict, completion_tags_selector: list[str], comment: str = None) -> httpx.Response: """ Example: >>> from log10.feedback import Feedback >>> fb = Feedback() >>> fb.create(task_id="task_id", rate={...}) """ - json_payload = {"task_id": task_id, "rate": rate} + json_payload = {"task_id": task_id, "json_values": rate, "completion_tags_selector": completion_tags_selector} res = self._post_request(self.feedback_create_url, json_payload) return res diff --git a/log10/feedback/feedback_task.py b/log10/feedback/feedback_task.py index d2319d5b..8990002c 100644 --- a/log10/feedback/feedback_task.py +++ b/log10/feedback/feedback_task.py @@ -29,9 +29,6 @@ def _post_request(self, url: str, json_payload: dict) -> httpx.Response: headers = {"x-log10-token": self._log10_config.token, "Content-Type": "application/json", "x-log10-organization": self._log10_config.org_id} json_payload["organization_id"] = self._log10_config.org_id try: - from pprint import pprint - pprint(f"{headers=}") - pprint(f"{json_payload=}") res = self._http_client.post(self._log10_config.url + url, headers=headers, json=json_payload) res.raise_for_status() return res @@ -46,7 +43,7 @@ def create(self, task_schema: dict, name: str = None, instruction: str = None) - >>> feedback_task = FeedbackTask() >>> task = feedback_task.create(name="summarization", task_schema={...}) """ - json_payload = {"task_schema": task_schema} + json_payload = {"json_schema": task_schema} if name: json_payload["name"] = name if instruction: From 7eb48ff903ba78a2e76198101c2c438761590747 Mon Sep 17 00:00:00 2001 From: Wenzhe Xue Date: Mon, 12 Feb 2024 15:52:58 -0800 Subject: [PATCH 06/12] make cli work --- log10/feedback/feedback.py | 47 +++++++++++++++------------------ log10/feedback/feedback_task.py | 23 +++++++--------- 2 files changed, 32 insertions(+), 38 deletions(-) diff --git a/log10/feedback/feedback.py b/log10/feedback/feedback.py index df95a8a5..890209ff 100644 --- a/log10/feedback/feedback.py +++ b/log10/feedback/feedback.py @@ -1,3 +1,4 @@ +import json import logging import click @@ -7,15 +8,6 @@ from log10.llm import Log10Config -# def create(name: str, task_schema: dict) -> httpx.Response: -# """ -# Example: -# >>> from log10.feedback import feedback, feedback_task -# >>> task = feedback_task.create(name="summarization", task_schema={...}) -# >>> task_id = task.id -# >>> fb = feedback.create(task_id=task_id, rate={...}) -# """ - load_dotenv() logging.basicConfig( @@ -34,7 +26,11 @@ def __init__(self, log10_config: Log10Config = None): self._http_client = httpx.Client() def _post_request(self, url: str, json_payload: dict) -> httpx.Response: - headers = {"x-log10-token": self._log10_config.token, "x-log10-organization": self._log10_config.org_id, "Content-Type": "application/json"} + headers = { + "x-log10-token": self._log10_config.token, + "x-log10-organization": self._log10_config.org_id, + "Content-Type": "application/json", + } json_payload["organization_id"] = self._log10_config.org_id try: res = self._http_client.post(self._log10_config.url + url, headers=headers, json=json_payload) @@ -42,27 +38,28 @@ def _post_request(self, url: str, json_payload: dict) -> httpx.Response: return res except Exception as e: logger.error(e) + logger.error(e.response.json()["error"]) raise - def create(self, task_id: str, rate: dict, completion_tags_selector: list[str], comment: str = None) -> httpx.Response: - """ - Example: - >>> from log10.feedback import Feedback - >>> fb = Feedback() - >>> fb.create(task_id="task_id", rate={...}) - """ - json_payload = {"task_id": task_id, "json_values": rate, "completion_tags_selector": completion_tags_selector} + def create( + self, task_id: str, values: dict, completion_tags_selector: list[str], comment: str = None + ) -> httpx.Response: + json_payload = { + "task_id": task_id, + "json_values": values, + "completion_tags_selector": completion_tags_selector, + } res = self._post_request(self.feedback_create_url, json_payload) return res @click.command() @click.option("--task_id", prompt="Enter task id", help="Task ID") -@click.option("--rate", prompt="Enter task rate", help="Rate in JSON format") -def create_feedback(task_id, rate): +@click.option("--values", prompt="Enter task values", help="Feedback in JSON format") +@click.option("--completion_tags_selector", prompt="Enter completion tags selector", help="Completion tags selector") +def create_feedback(task_id, values, completion_tags_selector): click.echo("Creating feedback") - click.echo(f"Task ID: {task_id}") - click.echo(f"Rate: {rate}") - # fb = Feedback() - # feedback = fb.create(task_id=task_id, rate=rate) - # click.echo(feedback) + tags = completion_tags_selector.split(",") + values = json.loads(values) + feedback = Feedback().create(task_id=task_id, values=values, completion_tags_selector=tags) + click.echo(feedback.json()) diff --git a/log10/feedback/feedback_task.py b/log10/feedback/feedback_task.py index 8990002c..393817fc 100644 --- a/log10/feedback/feedback_task.py +++ b/log10/feedback/feedback_task.py @@ -1,3 +1,4 @@ +import json import logging import click @@ -17,7 +18,6 @@ logger.setLevel(logging.INFO) - class FeedbackTask: feedback_task_create_url = "api/v1/feedback_task" @@ -26,7 +26,11 @@ def __init__(self, log10_config: Log10Config = None): self._http_client = httpx.Client() def _post_request(self, url: str, json_payload: dict) -> httpx.Response: - headers = {"x-log10-token": self._log10_config.token, "Content-Type": "application/json", "x-log10-organization": self._log10_config.org_id} + headers = { + "x-log10-token": self._log10_config.token, + "Content-Type": "application/json", + "x-log10-organization": self._log10_config.org_id, + } json_payload["organization_id"] = self._log10_config.org_id try: res = self._http_client.post(self._log10_config.url + url, headers=headers, json=json_payload) @@ -34,15 +38,10 @@ def _post_request(self, url: str, json_payload: dict) -> httpx.Response: return res except Exception as e: logger.error(e) + logger.error(e.response.json()["error"]) raise def create(self, task_schema: dict, name: str = None, instruction: str = None) -> httpx.Response: - """ - Example: - >>> from log10.feedback.feedback_task import FeedbackTask - >>> feedback_task = FeedbackTask() - >>> task = feedback_task.create(name="summarization", task_schema={...}) - """ json_payload = {"json_schema": task_schema} if name: json_payload["name"] = name @@ -59,8 +58,6 @@ def create(self, task_schema: dict, name: str = None, instruction: str = None) - @click.option("--task_schema", prompt="Enter feedback task schema", help="Task schema") def create_feedback_task(name, task_schema): click.echo("Creating feedback task") - click.echo(f"Name: {name}") - click.echo(f"Task Schema: {task_schema}") - # fb_task = FeedbackTask() - # task = fb_task.create(name=name, task_schema=task_schema) - # click.echo(task) + task_schema = json.loads(task_schema) + task = FeedbackTask().create(name=name, task_schema=task_schema) + click.echo(f"Use this task_id to add feedback: {task.json()['id']}") From 92b0f6450626990675a86f43881a8bc91ba36ef1 Mon Sep 17 00:00:00 2001 From: Wenzhe Xue Date: Mon, 12 Feb 2024 16:15:29 -0800 Subject: [PATCH 07/12] update examples/feedback/simple_feedback.py --- examples/feedback/create_feedback_task.py | 32 ----------------- examples/feedback/simple_feedback.py | 42 +++++++++++++++-------- 2 files changed, 28 insertions(+), 46 deletions(-) delete mode 100644 examples/feedback/create_feedback_task.py diff --git a/examples/feedback/create_feedback_task.py b/examples/feedback/create_feedback_task.py deleted file mode 100644 index 967ac524..00000000 --- a/examples/feedback/create_feedback_task.py +++ /dev/null @@ -1,32 +0,0 @@ -import logging -from pydantic import BaseModel, Field -from typing import Literal -from log10.feedback.feedback_task import FeedbackTask -from log10.feedback.feedback import Feedback - -httpx_logger = logging.getLogger("httpx") -httpx_logger.setLevel(logging.DEBUG) - -feedback_task = FeedbackTask() -t_s = { - "type": "object", - "properties": { - "feedback": { - "type": "string", - "enum": ["😀", "😬", "😐", "🙁", "😫"] - } - } -} -class EmojiFeedback(BaseModel): - feedback: Literal["😀", "🙁"] = Field(..., description="User feedback with emojis") - -eft = EmojiFeedback(feedback="😀") - -# convert t_s to json -task = feedback_task.create(name="emo", task_schema=eft.model_json_schema()) -task_dump = task.json() -print(task_dump["id"]) - -fb = Feedback() -print(eft.model_dump_json()) -fb.create(task_id=task_dump["id"], rate=eft.model_dump(), completion_tags_selector=["give_me_feedback"]) \ No newline at end of file diff --git a/examples/feedback/simple_feedback.py b/examples/feedback/simple_feedback.py index 9a57d43e..fc4c9888 100644 --- a/examples/feedback/simple_feedback.py +++ b/examples/feedback/simple_feedback.py @@ -1,3 +1,4 @@ +import uuid from pprint import pprint from typing import Literal @@ -8,11 +9,14 @@ from log10.load import OpenAI -class EmojiFeedback(BaseModel): - feedback: Literal["😀", "🙁"] = Field(..., description="User feedback with emojis") - +# +# use log10 to log an openai completion +# -client = OpenAI() +# create a unique id +unique_id = str(uuid.uuid4()) +print(f"Use tag: {unique_id}") +client = OpenAI(tags=[unique_id]) completion = client.chat.completions.create( model="gpt-3.5-turbo", messages=[ @@ -26,13 +30,23 @@ class EmojiFeedback(BaseModel): }, ], ) -completion_id = completion.choices[0].id - -task = FeedbackTask() -res = task.create(name="emoji_feedback_task", task_schema=EmojiFeedback.model_json_schema()) -task_id = res["id"] -pprint(task) -# Example usage -fb = Feedback() -res = fb.create(task_id=task_id, data=EmojiFeedback(feedback="😀").model_dump_json(), completion_tags_selector=[str(completion_id)]) -pprint(res) +print(completion.choices[0].message) + +# +# add feedback to the completion +# + + +# define a feedback task +class EmojiFeedback(BaseModel): + feedback: Literal["😀", "🙁"] = Field(..., description="User feedback with emojis") + + +# create a feedback +fb = EmojiFeedback(feedback="😀") + +task = FeedbackTask().create(name="emoji_task_test", task_schema=fb.model_json_schema()) +task_dump = task.json() + +print(fb.model_dump_json()) +Feedback().create(task_id=task_dump["id"], values=fb.model_dump(), completion_tags_selector=[unique_id]) From 10817f8ec8864685fe0e3dda10c2563bc68c212b Mon Sep 17 00:00:00 2001 From: Wenzhe Xue Date: Mon, 12 Feb 2024 16:17:43 -0800 Subject: [PATCH 08/12] remove unused lib --- examples/feedback/simple_feedback.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/feedback/simple_feedback.py b/examples/feedback/simple_feedback.py index fc4c9888..a1417113 100644 --- a/examples/feedback/simple_feedback.py +++ b/examples/feedback/simple_feedback.py @@ -1,5 +1,4 @@ import uuid -from pprint import pprint from typing import Literal from pydantic import BaseModel, Field From ec280400631acdf555b29a6fe32c3a834da56e99 Mon Sep 17 00:00:00 2001 From: wenzhe <145375501+wenzhe-log10@users.noreply.github.com> Date: Mon, 12 Feb 2024 16:26:04 -0800 Subject: [PATCH 09/12] Update log10/feedback/feedback.py Co-authored-by: Kim Tran <17498395+kxtran@users.noreply.github.com> --- log10/feedback/feedback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/log10/feedback/feedback.py b/log10/feedback/feedback.py index 890209ff..f29d3b34 100644 --- a/log10/feedback/feedback.py +++ b/log10/feedback/feedback.py @@ -28,7 +28,7 @@ def __init__(self, log10_config: Log10Config = None): def _post_request(self, url: str, json_payload: dict) -> httpx.Response: headers = { "x-log10-token": self._log10_config.token, - "x-log10-organization": self._log10_config.org_id, + "x-log10-organization-id": self._log10_config.org_id, "Content-Type": "application/json", } json_payload["organization_id"] = self._log10_config.org_id From 933e8f365c367c1eac4cc9c762b0c44d4bfcbd14 Mon Sep 17 00:00:00 2001 From: wenzhe <145375501+wenzhe-log10@users.noreply.github.com> Date: Mon, 12 Feb 2024 16:26:10 -0800 Subject: [PATCH 10/12] Update log10/feedback/feedback_task.py Co-authored-by: Kim Tran <17498395+kxtran@users.noreply.github.com> --- log10/feedback/feedback_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/log10/feedback/feedback_task.py b/log10/feedback/feedback_task.py index 393817fc..cda86bde 100644 --- a/log10/feedback/feedback_task.py +++ b/log10/feedback/feedback_task.py @@ -29,7 +29,7 @@ def _post_request(self, url: str, json_payload: dict) -> httpx.Response: headers = { "x-log10-token": self._log10_config.token, "Content-Type": "application/json", - "x-log10-organization": self._log10_config.org_id, + "x-log10-organization-id": self._log10_config.org_id, } json_payload["organization_id"] = self._log10_config.org_id try: From 79dde66bc463d83ba7818f6aea569e0630afaefc Mon Sep 17 00:00:00 2001 From: Wenzhe Xue Date: Mon, 12 Feb 2024 16:39:44 -0800 Subject: [PATCH 11/12] update README --- README.md | 5 +++++ examples/feedback/simple_feedback.py | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b09a290e..5a701648 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,11 @@ Read more here for options for logging using library wrapper, langchain callback Optimizing prompts requires a lot of manual effort. Log10 provides a copilot that can help you with suggestions on how to [optimize your prompt](https://log10.io/docs/prompt_engineering/auto_prompt#how-to-use-auto-prompting-in-log10-python-library). +### 👷🔢 Feedback + +Add feedback to your completions. Checkout the Python [example](/examples/feedback/simple_feedback.py) +or use CLI `log10 feedback-task create` and `log10 feedback create`. Please check our [doc](https://log10.io/docs/feedback) for more details. + ### 🔍🐞 Prompt chain debugging Prompt chains such as those in [Langchain](https://github.com/hwchase17/langchain) can be difficult to debug. Log10 provides prompt provenance, session tracking and call stack functionality to help debug chains. diff --git a/examples/feedback/simple_feedback.py b/examples/feedback/simple_feedback.py index a1417113..36ac656a 100644 --- a/examples/feedback/simple_feedback.py +++ b/examples/feedback/simple_feedback.py @@ -35,7 +35,6 @@ # add feedback to the completion # - # define a feedback task class EmojiFeedback(BaseModel): feedback: Literal["😀", "🙁"] = Field(..., description="User feedback with emojis") From 56005be08507b81d5287eee6a6412fe96e287151 Mon Sep 17 00:00:00 2001 From: Wenzhe Xue Date: Mon, 12 Feb 2024 16:44:02 -0800 Subject: [PATCH 12/12] format --- examples/feedback/simple_feedback.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/feedback/simple_feedback.py b/examples/feedback/simple_feedback.py index 36ac656a..a1417113 100644 --- a/examples/feedback/simple_feedback.py +++ b/examples/feedback/simple_feedback.py @@ -35,6 +35,7 @@ # add feedback to the completion # + # define a feedback task class EmojiFeedback(BaseModel): feedback: Literal["😀", "🙁"] = Field(..., description="User feedback with emojis")