From d3617416a8fd5fc5b2f020a05eb3c53434e6f248 Mon Sep 17 00:00:00 2001 From: Kim Tran Date: Tue, 25 Jun 2024 18:20:35 -0400 Subject: [PATCH] Move cli code into its own folder --- log10/__main__.py | 8 +- log10/feedback/autofeedback.py | 43 --------- log10/feedback/cli/autofeedback.py | 47 ++++++++++ log10/feedback/cli/feedback.py | 129 +++++++++++++++++++++++++++ log10/feedback/cli/feedback_task.py | 80 +++++++++++++++++ log10/feedback/feedback.py | 130 +--------------------------- log10/feedback/feedback_task.py | 78 +---------------- 7 files changed, 263 insertions(+), 252 deletions(-) create mode 100644 log10/feedback/cli/autofeedback.py create mode 100644 log10/feedback/cli/feedback.py create mode 100644 log10/feedback/cli/feedback_task.py diff --git a/log10/__main__.py b/log10/__main__.py index ca086d1e..bd9a4bcf 100644 --- a/log10/__main__.py +++ b/log10/__main__.py @@ -2,14 +2,14 @@ import click except ImportError: print( - "To use cli you must install optional modules. Please install them with `pip install 'log10-io[cliModules]'`." + "To use log10 cli you must install optional modules. Please install them with `pip install 'log10-io[cliModules]'`." ) exit(1) from log10.completions.completions import benchmark_models, download_completions, get_completion, list_completions -from log10.feedback.autofeedback import auto_feedback_icl, get_autofeedback_cli -from log10.feedback.feedback import create_feedback, download_feedback, get_feedback, list_feedback -from log10.feedback.feedback_task import create_feedback_task, get_feedback_task, list_feedback_task +from log10.feedback.cli.autofeedback import auto_feedback_icl, get_autofeedback_cli +from log10.feedback.cli.feedback import create_feedback, download_feedback, get_feedback, list_feedback +from log10.feedback.cli.feedback_task import create_feedback_task, get_feedback_task, list_feedback_task @click.group() diff --git a/log10/feedback/autofeedback.py b/log10/feedback/autofeedback.py index e435bd13..d6e0e353 100644 --- a/log10/feedback/autofeedback.py +++ b/log10/feedback/autofeedback.py @@ -3,10 +3,7 @@ import random from types import FunctionType -import click import httpx -import rich -from rich.console import Console from log10._httpx_utils import _try_post_graphql_request from log10.completions.completions import _get_completion @@ -130,43 +127,3 @@ def get_autofeedback(completion_id: str) -> httpx.Response: return response.json() else: response.raise_for_status() - - -@click.command() -@click.option("--task_id", help="Feedback task ID") -@click.option("--content", help="Completion content") -@click.option("--file", "-f", help="File containing completion content") -@click.option("--completion_id", help="Completion ID") -@click.option("--num_samples", default=5, help="Number of samples to use for few-shot learning") -def auto_feedback_icl(task_id: str, content: str, file: str, completion_id: str, num_samples: int): - """ - Generate feedback with existing human feedback based on in context learning - """ - options_count = sum([1 for option in [content, file, completion_id] if option]) - if options_count > 1: - click.echo("Only one of --content, --file, or --completion_id should be provided.") - return - - console = Console() - auto_feedback_icl = AutoFeedbackICL(task_id, num_samples=num_samples) - if completion_id: - results = auto_feedback_icl.predict(completion_id=completion_id) - console.print_json(results) - return - - if file: - with open(file, "r") as f: - content = f.read() - results = auto_feedback_icl.predict(text=content) - console.print_json(results) - - -@click.command() -@click.option("--completion-id", required=True, help="Completion ID") -def get_autofeedback_cli(completion_id: str): - """ - Get an auto feedback by completion id - """ - res = get_autofeedback(completion_id) - if res: - rich.print_json(json.dumps(res["data"], indent=4)) diff --git a/log10/feedback/cli/autofeedback.py b/log10/feedback/cli/autofeedback.py new file mode 100644 index 00000000..a1ecf364 --- /dev/null +++ b/log10/feedback/cli/autofeedback.py @@ -0,0 +1,47 @@ +import json + +import click +import rich +from rich.console import Console + +from log10.feedback.autofeedback import AutoFeedbackICL, get_autofeedback + + +@click.command() +@click.option("--task_id", help="Feedback task ID") +@click.option("--content", help="Completion content") +@click.option("--file", "-f", help="File containing completion content") +@click.option("--completion_id", help="Completion ID") +@click.option("--num_samples", default=5, help="Number of samples to use for few-shot learning") +def auto_feedback_icl(task_id: str, content: str, file: str, completion_id: str, num_samples: int): + """ + Generate feedback with existing human feedback based on in context learning + """ + options_count = sum([1 for option in [content, file, completion_id] if option]) + if options_count > 1: + click.echo("Only one of --content, --file, or --completion_id should be provided.") + return + + console = Console() + auto_feedback_icl = AutoFeedbackICL(task_id, num_samples=num_samples) + if completion_id: + results = auto_feedback_icl.predict(completion_id=completion_id) + console.print_json(results) + return + + if file: + with open(file, "r") as f: + content = f.read() + results = auto_feedback_icl.predict(text=content) + console.print_json(results) + + +@click.command() +@click.option("--completion-id", required=True, help="Completion ID") +def get_autofeedback_cli(completion_id: str): + """ + Get an auto feedback by completion id + """ + res = get_autofeedback(completion_id) + if res: + rich.print_json(json.dumps(res["data"], indent=4)) diff --git a/log10/feedback/cli/feedback.py b/log10/feedback/cli/feedback.py new file mode 100644 index 00000000..f8398b77 --- /dev/null +++ b/log10/feedback/cli/feedback.py @@ -0,0 +1,129 @@ +import json + +import click +from rich.console import Console +from rich.table import Table +from tqdm import tqdm + +from log10.feedback.feedback import Feedback, _get_feedback_list + + +@click.command() +@click.option("--task_id", prompt="Enter task id", help="Task ID") +@click.option("--values", prompt="Enter task values", help="Feedback in JSON format") +@click.option( + "--completion_tags_selector", + prompt="Enter completion tags selector", + help="Completion tags selector", +) +@click.option("--comment", help="Comment", default="") +def create_feedback(task_id, values, completion_tags_selector, comment): + """ + Add feedback to a group of completions associated with a task + """ + click.echo("Creating feedback") + tags = completion_tags_selector.split(",") + values = json.loads(values) + feedback = Feedback().create(task_id=task_id, values=values, completion_tags_selector=tags, comment=comment) + click.echo(feedback.json()) + + +@click.command() +@click.option( + "--offset", default=0, type=int, help="The starting index from which to begin the feedback fetch. Defaults to 0." +) +@click.option( + "--limit", default=25, type=int, help="The maximum number of feedback items to retrieve. Defaults to 25." +) +@click.option( + "--task_id", + default="", + type=str, + help="The specific Task ID to filter feedback. If not provided, feedback for all tasks will be fetched.", +) +def list_feedback(offset, limit, task_id): + """ + List feedback based on the provided criteria. This command allows fetching feedback for a specific task or across all tasks, + with control over the starting point and the number of items to retrieve. + """ + feedback_data = _get_feedback_list(offset, limit, task_id) + data_for_table = [] + for feedback in feedback_data: + data_for_table.append( + { + "id": feedback["id"], + "task_name": feedback["task_name"], + "feedback": json.dumps(feedback["json_values"], ensure_ascii=False), + "matched_completion_ids": ",".join(feedback["matched_completion_ids"]), + } + ) + table = Table(title="Feedback") + table.add_column("ID") + table.add_column("Task Name") + table.add_column("Feedback") + table.add_column("Completion ID") + + for item in data_for_table: + table.add_row(item["id"], item["task_name"], item["feedback"], item["matched_completion_ids"]) + console = Console() + console.print(table) + console.print(f"Total feedback: {len(feedback_data)}") + + +@click.command() +@click.option("--id", required=True, help="Get feedback by ID") +def get_feedback(id): + """ + Get feedback based on provided ID. + """ + try: + res = Feedback().get(id) + except Exception as e: + click.echo(f"Error fetching feedback {e}") + if hasattr(e, "response") and hasattr(e.response, "json") and "error" in e.response.json(): + click.echo(e.response.json()["error"]) + return + console = Console() + feedback = json.dumps(res.json(), indent=4) + console.print_json(feedback) + + +@click.command() +@click.option( + "--offset", + default=0, + help="The starting index from which to begin the feedback fetch. Leave empty to start from the beginning.", +) +@click.option( + "--limit", default="", help="The maximum number of feedback items to retrieve. Leave empty to retrieve all." +) +@click.option( + "--task_id", + default="", + type=str, + help="The specific Task ID to filter feedback. If not provided, feedback for all tasks will be fetched.", +) +@click.option( + "--file", + "-f", + type=str, + required=False, + help="Path to the file where the feedback will be saved. The feedback data is saved in JSON Lines (jsonl) format. If not specified, feedback will be printed to stdout.", +) +def download_feedback(offset, limit, task_id, file): + """ + Download feedback based on the provided criteria. This command allows fetching feedback for a specific task or across all tasks, + with control over the starting point and the number of items to retrieve. + """ + feedback_data = _get_feedback_list(offset, limit, task_id) + + console = Console() + if not file: + for feedback in feedback_data: + console.print_json(json.dumps(feedback, indent=4)) + return + + with open(file, "w") as f: + console.print(f"Saving feedback to {file}") + for feedback in tqdm(feedback_data): + f.write(json.dumps(feedback) + "\n") diff --git a/log10/feedback/cli/feedback_task.py b/log10/feedback/cli/feedback_task.py new file mode 100644 index 00000000..056730d3 --- /dev/null +++ b/log10/feedback/cli/feedback_task.py @@ -0,0 +1,80 @@ +import json + +import click +from rich.console import Console +from rich.table import Table + +from log10._httpx_utils import _get_time_diff +from log10.feedback.feedback_task import FeedbackTask + + +# create a cli interface for FeebackTask.create function +@click.command() +@click.option("--name", prompt="Enter feedback task name", help="Name of the task") +@click.option("--task_schema", prompt="Enter feedback task schema", help="Task schema") +@click.option("--instruction", help="Task instruction", default="") +@click.option( + "--completion_tags_selector", + help="Completion tags selector", +) +def create_feedback_task(name, task_schema, instruction, completion_tags_selector=None): + click.echo("Creating feedback task") + tags = [] + + if completion_tags_selector: + tags = completion_tags_selector.split(",") + + task_schema = json.loads(task_schema) + task = FeedbackTask().create( + name=name, task_schema=task_schema, completion_tags_selector=tags, instruction=instruction + ) + click.echo(f"Use this task_id to add feedback: {task.json()['id']}") + + +@click.command() +@click.option("--limit", default=25, help="Number of feedback tasks to fetch") +@click.option("--offset", default=0, help="Offset for the feedback tasks") +def list_feedback_task(limit, offset): + res = FeedbackTask().list(limit=limit, offset=offset) + feedback_tasks = res.json() + + data_for_table = [] + + for task in feedback_tasks["data"]: + data_for_table.append( + { + "id": task["id"], + "created_at": _get_time_diff(task["created_at"]), + "name": task["name"], + "required": task["json_schema"]["required"], + "instruction": task["instruction"], + } + ) + + table = Table(title="Feedback Tasks") + table.add_column("ID", style="dim") + table.add_column("Created At") + table.add_column("Name") + table.add_column("Required") + table.add_column("Instruction") + for item in data_for_table: + required = ", ".join(item["required"]) if item["required"] else "" + table.add_row(item["id"], item["created_at"], item["name"], required, item["instruction"]) + + console = Console() + console.print(table) + + +@click.command() +@click.option("--id", help="Get feedback task by ID") +def get_feedback_task(id): + try: + res = FeedbackTask().get(id) + except Exception as e: + click.echo(f"Error fetching feedback task {e}") + if hasattr(e, "response") and hasattr(e.response, "json") and "error" in e.response.json(): + click.echo(e.response.json()["error"]) + return + task = json.dumps(res.json()) + console = Console() + console.print_json(task) diff --git a/log10/feedback/feedback.py b/log10/feedback/feedback.py index 08c15c90..1f2b557e 100644 --- a/log10/feedback/feedback.py +++ b/log10/feedback/feedback.py @@ -1,11 +1,6 @@ -import json import logging -import click import httpx -from rich.console import Console -from rich.table import Table -from tqdm import tqdm from log10._httpx_utils import _try_get from log10.llm import Log10Config @@ -86,26 +81,6 @@ def get(self, id: str) -> httpx.Response: return res -@click.command() -@click.option("--task_id", prompt="Enter task id", help="Task ID") -@click.option("--values", prompt="Enter task values", help="Feedback in JSON format") -@click.option( - "--completion_tags_selector", - prompt="Enter completion tags selector", - help="Completion tags selector", -) -@click.option("--comment", help="Comment", default="") -def create_feedback(task_id, values, completion_tags_selector, comment): - """ - Add feedback to a group of completions associated with a task - """ - click.echo("Creating feedback") - tags = completion_tags_selector.split(",") - values = json.loads(values) - feedback = Feedback().create(task_id=task_id, values=values, completion_tags_selector=tags, comment=comment) - click.echo(feedback.json()) - - def _get_feedback_list(offset, limit, task_id): total_fetched = 0 feedback_data = [] @@ -130,110 +105,9 @@ def _get_feedback_list(offset, limit, task_id): if total_fetched >= limit or total_fetched >= total_feedback: break except Exception as e: - click.echo(f"Error fetching feedback {e}") + logger.error(f"Error fetching feedback {e}") if hasattr(e, "response") and hasattr(e.response, "json") and "error" in e.response.json(): - click.echo(e.response.json()["error"]) + logger.error(e.response.json()["error"]) return [] return feedback_data - - -@click.command() -@click.option( - "--offset", default=0, type=int, help="The starting index from which to begin the feedback fetch. Defaults to 0." -) -@click.option( - "--limit", default=25, type=int, help="The maximum number of feedback items to retrieve. Defaults to 25." -) -@click.option( - "--task_id", - default="", - type=str, - help="The specific Task ID to filter feedback. If not provided, feedback for all tasks will be fetched.", -) -def list_feedback(offset, limit, task_id): - """ - List feedback based on the provided criteria. This command allows fetching feedback for a specific task or across all tasks, - with control over the starting point and the number of items to retrieve. - """ - feedback_data = _get_feedback_list(offset, limit, task_id) - data_for_table = [] - for feedback in feedback_data: - data_for_table.append( - { - "id": feedback["id"], - "task_name": feedback["task_name"], - "feedback": json.dumps(feedback["json_values"], ensure_ascii=False), - "matched_completion_ids": ",".join(feedback["matched_completion_ids"]), - } - ) - table = Table(title="Feedback") - table.add_column("ID") - table.add_column("Task Name") - table.add_column("Feedback") - table.add_column("Completion ID") - - for item in data_for_table: - table.add_row(item["id"], item["task_name"], item["feedback"], item["matched_completion_ids"]) - console = Console() - console.print(table) - console.print(f"Total feedback: {len(feedback_data)}") - - -@click.command() -@click.option("--id", required=True, help="Get feedback by ID") -def get_feedback(id): - """ - Get feedback based on provided ID. - """ - try: - res = Feedback().get(id) - except Exception as e: - click.echo(f"Error fetching feedback {e}") - if hasattr(e, "response") and hasattr(e.response, "json") and "error" in e.response.json(): - click.echo(e.response.json()["error"]) - return - console = Console() - feedback = json.dumps(res.json(), indent=4) - console.print_json(feedback) - - -@click.command() -@click.option( - "--offset", - default=0, - help="The starting index from which to begin the feedback fetch. Leave empty to start from the beginning.", -) -@click.option( - "--limit", default="", help="The maximum number of feedback items to retrieve. Leave empty to retrieve all." -) -@click.option( - "--task_id", - default="", - type=str, - help="The specific Task ID to filter feedback. If not provided, feedback for all tasks will be fetched.", -) -@click.option( - "--file", - "-f", - type=str, - required=False, - help="Path to the file where the feedback will be saved. The feedback data is saved in JSON Lines (jsonl) format. If not specified, feedback will be printed to stdout.", -) -def download_feedback(offset, limit, task_id, file): - """ - Download feedback based on the provided criteria. This command allows fetching feedback for a specific task or across all tasks, - with control over the starting point and the number of items to retrieve. - """ - feedback_data = _get_feedback_list(offset, limit, task_id) - - console = Console() - if not file: - for feedback in feedback_data: - console.print_json(json.dumps(feedback, indent=4)) - return - - with open(file, "w") as f: - console.print(f"Saving feedback to {file}") - for feedback in tqdm(feedback_data): - f.write(json.dumps(feedback) + "\n") diff --git a/log10/feedback/feedback_task.py b/log10/feedback/feedback_task.py index 53c91049..9c75c3b8 100644 --- a/log10/feedback/feedback_task.py +++ b/log10/feedback/feedback_task.py @@ -1,13 +1,9 @@ -import json import logging -import click import httpx from dotenv import load_dotenv -from rich.console import Console -from rich.table import Table -from log10._httpx_utils import _get_time_diff, _try_get +from log10._httpx_utils import _try_get from log10.llm import Log10Config @@ -74,75 +70,3 @@ def get(self, id: str) -> httpx.Response: if res.status_code != 200: raise Exception(f"Error fetching feedback task {res.json()}") return res - - -# create a cli interface for FeebackTask.create function -@click.command() -@click.option("--name", prompt="Enter feedback task name", help="Name of the task") -@click.option("--task_schema", prompt="Enter feedback task schema", help="Task schema") -@click.option("--instruction", help="Task instruction", default="") -@click.option( - "--completion_tags_selector", - help="Completion tags selector", -) -def create_feedback_task(name, task_schema, instruction, completion_tags_selector=None): - click.echo("Creating feedback task") - tags = [] - - if completion_tags_selector: - tags = completion_tags_selector.split(",") - - task_schema = json.loads(task_schema) - task = FeedbackTask().create( - name=name, task_schema=task_schema, completion_tags_selector=tags, instruction=instruction - ) - click.echo(f"Use this task_id to add feedback: {task.json()['id']}") - - -@click.command() -@click.option("--limit", default=25, help="Number of feedback tasks to fetch") -@click.option("--offset", default=0, help="Offset for the feedback tasks") -def list_feedback_task(limit, offset): - res = FeedbackTask().list(limit=limit, offset=offset) - feedback_tasks = res.json() - - data_for_table = [] - - for task in feedback_tasks["data"]: - data_for_table.append( - { - "id": task["id"], - "created_at": _get_time_diff(task["created_at"]), - "name": task["name"], - "required": task["json_schema"]["required"], - "instruction": task["instruction"], - } - ) - - table = Table(title="Feedback Tasks") - table.add_column("ID", style="dim") - table.add_column("Created At") - table.add_column("Name") - table.add_column("Required") - table.add_column("Instruction") - for item in data_for_table: - required = ", ".join(item["required"]) if item["required"] else "" - table.add_row(item["id"], item["created_at"], item["name"], required, item["instruction"]) - - console = Console() - console.print(table) - - -@click.command() -@click.option("--id", help="Get feedback task by ID") -def get_feedback_task(id): - try: - res = FeedbackTask().get(id) - except Exception as e: - click.echo(f"Error fetching feedback task {e}") - if hasattr(e, "response") and hasattr(e.response, "json") and "error" in e.response.json(): - click.echo(e.response.json()["error"]) - return - task = json.dumps(res.json()) - console = Console() - console.print_json(task)