diff --git a/cli_docs.md b/cli_docs.md index 268dbd12..a1943d10 100644 --- a/cli_docs.md +++ b/cli_docs.md @@ -194,6 +194,8 @@ And to download to a JSONL file, use `log10 feedback download --task_id 04405cbc To leverage your feedback and use AI to generate more feedback automatically, use [`log10 feedback predict`](#log10-feedback-predict). Please refer to this [doc](https://log10.io/docs/feedback/auto_feedback) for more details. +To get auto generated feedback for a completion, use [`log10 feedback autofeedback get`](#log10-feedback-autofeedback-get) + ## CLI References ### Completions @@ -399,6 +401,8 @@ Options: $ log10 feedback predict --help Usage: log10 feedback predict [OPTIONS] + Generate feedback with existing human feedback based on in context learning + Options: --task_id TEXT Feedback task ID --content TEXT Completion content @@ -407,6 +411,17 @@ Options: --num_samples INTEGER Number of samples to use for few-shot learning ``` +#### log10 feedback autofeedback get +```bash +$ log10 feedback autofeedback get --help +Usage: log10 feedback autofeedback get [OPTIONS] + + Get an auto feedback by completion id + +Options: + --completion-id TEXT Completion ID [required] +``` + ### Feedback Task ```bash diff --git a/log10/__main__.py b/log10/__main__.py index 20a67310..6365121b 100644 --- a/log10/__main__.py +++ b/log10/__main__.py @@ -1,7 +1,7 @@ import click from log10.completions.completions import benchmark_models, download_completions, get_completion, list_completions -from log10.feedback.autofeedback import auto_feedback_icl +from log10.feedback.autofeedback import auto_feedback_icl, get_autofeedback_cli from log10.feedback.feedback import create_feedback, download_feedback, get_feedback, list_feedback from log10.feedback.feedback_task import create_feedback_task, get_feedback_task, list_feedback_task @@ -19,7 +19,7 @@ def completions(): pass -@click.group() +@click.group(name="feedback") def feedback(): """ Manage feedback for completions i.e. capturing feedback from users @@ -27,6 +27,14 @@ def feedback(): pass +@click.group(name="auto_feedback") +def auto_feedback(): + """ + Manage auto feedback for completions i.e. capturing feedback from users + """ + pass + + @click.group() def feedback_task(): """ @@ -47,6 +55,9 @@ def feedback_task(): feedback.add_command(get_feedback, "get") feedback.add_command(download_feedback, "download") feedback.add_command(auto_feedback_icl, "predict") +feedback.add_command(auto_feedback, "autofeedback") +# Subcommands for auto_feedback under feedback command +auto_feedback.add_command(get_autofeedback_cli, "get") cli.add_command(feedback_task) feedback_task.add_command(create_feedback_task, "create") diff --git a/log10/_httpx_utils.py b/log10/_httpx_utils.py index cab1e1c5..00402561 100644 --- a/log10/_httpx_utils.py +++ b/log10/_httpx_utils.py @@ -14,6 +14,7 @@ logger: logging.Logger = logging.getLogger("LOG10") +GRAPHQL_URL = "https://graphql.log10.io/graphql" _log10_config = Log10Config() base_url = _log10_config.url @@ -85,6 +86,48 @@ def _try_post_request(url: str, payload: dict = {}) -> httpx.Response: logger.error(f"Failed to insert in log10: {payload} with error {err}") +def _try_post_graphql_request(query: str, variables: dict = {}) -> httpx.Response: + headers = {"content-type": "application/json", "x-api-token": _log10_config.token} + + payload = {"query": query, "variables": variables} + res = None + try: + res = httpx_client.post(GRAPHQL_URL, headers=headers, json=payload) + res.raise_for_status() + return res + except httpx.HTTPError as http_err: + if "401" in str(http_err): + logger.error( + "Failed anthorization. Please verify that LOG10_TOKEN and LOG10_ORG_ID are set correctly and try again." + + "\nSee https://github.com/log10-io/log10#%EF%B8%8F-setup for details" + ) + else: + logger.error(f"Failed with error: {http_err}") + except Exception as err: + logger.error(f"Failed to make requests to log10 graphQL: {payload} with error {err}") + + +async def _try_post_graphql_request_async(query: str, variables: dict = {}) -> httpx.Response: + headers = {"content-type": "application/json", "x-api-token": _log10_config.token} + + payload = {"query": query, "variables": variables} + res = None + try: + res = await httpx_async_client.post(GRAPHQL_URL, headers=headers, json=payload) + res.raise_for_status() + return res + except httpx.HTTPError as http_err: + if "401" in str(http_err): + logger.error( + "Failed anthorization. Please verify that LOG10_TOKEN and LOG10_ORG_ID are set correctly and try again." + + "\nSee https://github.com/log10-io/log10#%EF%B8%8F-setup for details" + ) + else: + logger.error(f"Failed with error: {http_err}") + except Exception as err: + logger.error(f"Failed to make requests to log10 graphQL: {payload} with error {err}") + + async def _try_post_request_async(url: str, payload: dict = {}) -> httpx.Response: headers = { "x-log10-token": _log10_config.token, diff --git a/log10/feedback/autofeedback.py b/log10/feedback/autofeedback.py index 20e4cdbb..e435bd13 100644 --- a/log10/feedback/autofeedback.py +++ b/log10/feedback/autofeedback.py @@ -4,10 +4,14 @@ from types import FunctionType import click +import httpx +import rich from rich.console import Console +from log10._httpx_utils import _try_post_graphql_request from log10.completions.completions import _get_completion from log10.feedback.feedback import _get_feedback_list +from log10.llm import Log10Config from log10.load import log10_session @@ -22,6 +26,8 @@ logger = logging.getLogger("LOG10") logger.setLevel(logging.INFO) +_log10_config = Log10Config() + class AutoFeedbackICL: """ @@ -95,6 +101,37 @@ def predict(self, text: str = None, completion_id: str = None) -> str: return ret +def get_autofeedback(completion_id: str) -> httpx.Response: + query = """ + query OrganizationCompletion($orgId: String!, $id: String!) { + organization(id: $orgId) { + completion(id: $id) { + id + autoFeedback { + id + status + jsonValues + comment + } + } + } + } + """ + + variables = {"orgId": _log10_config.org_id, "id": completion_id} + + response = _try_post_graphql_request(query, variables) + + if response is None: + logger.error(f"Failed to get auto feedback for completion {completion_id}") + return None + + if response.status_code == 200: + return response.json() + else: + response.raise_for_status() + + @click.command() @click.option("--task_id", help="Feedback task ID") @click.option("--content", help="Completion content") @@ -102,6 +139,9 @@ def predict(self, text: str = None, completion_id: str = None) -> str: @click.option("--completion_id", help="Completion ID") @click.option("--num_samples", default=5, help="Number of samples to use for few-shot learning") def auto_feedback_icl(task_id: str, content: str, file: str, completion_id: str, num_samples: int): + """ + Generate feedback with existing human feedback based on in context learning + """ options_count = sum([1 for option in [content, file, completion_id] if option]) if options_count > 1: click.echo("Only one of --content, --file, or --completion_id should be provided.") @@ -119,3 +159,14 @@ def auto_feedback_icl(task_id: str, content: str, file: str, completion_id: str, content = f.read() results = auto_feedback_icl.predict(text=content) console.print_json(results) + + +@click.command() +@click.option("--completion-id", required=True, help="Completion ID") +def get_autofeedback_cli(completion_id: str): + """ + Get an auto feedback by completion id + """ + res = get_autofeedback(completion_id) + if res: + rich.print_json(json.dumps(res["data"], indent=4)) diff --git a/log10/feedback/feedback.py b/log10/feedback/feedback.py index fd8e521a..08c15c90 100644 --- a/log10/feedback/feedback.py +++ b/log10/feedback/feedback.py @@ -181,7 +181,7 @@ def list_feedback(offset, limit, task_id): @click.command() -@click.option("--id", help="Get feedback by ID") +@click.option("--id", required=True, help="Get feedback by ID") def get_feedback(id): """ Get feedback based on provided ID.