Move cli code into its own folder

log10/__main__.py

@@ -2,14 +2,14 @@
  import click
 except ImportError:
  print(
- "To use cli you must install optional modules. Please install them with `pip install 'log10-io[cliModules]'`."
+ "To use log10 cli you must install optional modules. Please install them with `pip install 'log10-io[cliModules]'`."
  )
  exit(1)
 
 from log10.completions.completions import benchmark_models, download_completions, get_completion, list_completions
-from log10.feedback.autofeedback import auto_feedback_icl, get_autofeedback_cli
-from log10.feedback.feedback import create_feedback, download_feedback, get_feedback, list_feedback
-from log10.feedback.feedback_task import create_feedback_task, get_feedback_task, list_feedback_task
+from log10.feedback.cli.autofeedback import auto_feedback_icl, get_autofeedback_cli
+from log10.feedback.cli.feedback import create_feedback, download_feedback, get_feedback, list_feedback
+from log10.feedback.cli.feedback_task import create_feedback_task, get_feedback_task, list_feedback_task
 
 
 @click.group()

log10/feedback/autofeedback.py

@@ -3,10 +3,7 @@
 import random
 from types import FunctionType
 
-import click
 import httpx
-import rich
-from rich.console import Console
 
 from log10._httpx_utils import _try_post_graphql_request
 from log10.completions.completions import _get_completion
@@ -130,43 +127,3 @@ def get_autofeedback(completion_id: str) -> httpx.Response:
  return response.json()
  else:
  response.raise_for_status()
-
-
-@click.command()
-@click.option("--task_id", help="Feedback task ID")
-@click.option("--content", help="Completion content")
-@click.option("--file", "-f", help="File containing completion content")
-@click.option("--completion_id", help="Completion ID")
-@click.option("--num_samples", default=5, help="Number of samples to use for few-shot learning")
-def auto_feedback_icl(task_id: str, content: str, file: str, completion_id: str, num_samples: int):
- """
- Generate feedback with existing human feedback based on in context learning
- """
- options_count = sum([1 for option in [content, file, completion_id] if option])
- if options_count > 1:
- click.echo("Only one of --content, --file, or --completion_id should be provided.")
- return
-
- console = Console()
- auto_feedback_icl = AutoFeedbackICL(task_id, num_samples=num_samples)
- if completion_id:
- results = auto_feedback_icl.predict(completion_id=completion_id)
- console.print_json(results)
- return
-
- if file:
- with open(file, "r") as f:
- content = f.read()
- results = auto_feedback_icl.predict(text=content)
- console.print_json(results)
-
-
-@click.command()
-@click.option("--completion-id", required=True, help="Completion ID")
-def get_autofeedback_cli(completion_id: str):
- """
- Get an auto feedback by completion id
- """
- res = get_autofeedback(completion_id)
- if res:
- rich.print_json(json.dumps(res["data"], indent=4))

log10/feedback/cli/autofeedback.py

@@ -0,0 +1,47 @@
+import json
+
+import click
+import rich
+from rich.console import Console
+
+from log10.feedback.autofeedback import AutoFeedbackICL, get_autofeedback
+
+
+@click.command()
+@click.option("--task_id", help="Feedback task ID")
+@click.option("--content", help="Completion content")
+@click.option("--file", "-f", help="File containing completion content")
+@click.option("--completion_id", help="Completion ID")
+@click.option("--num_samples", default=5, help="Number of samples to use for few-shot learning")
+def auto_feedback_icl(task_id: str, content: str, file: str, completion_id: str, num_samples: int):
+ """
+ Generate feedback with existing human feedback based on in context learning
+ """
+ options_count = sum([1 for option in [content, file, completion_id] if option])
+ if options_count > 1:
+ click.echo("Only one of --content, --file, or --completion_id should be provided.")
+ return
+
+ console = Console()
+ auto_feedback_icl = AutoFeedbackICL(task_id, num_samples=num_samples)
+ if completion_id:
+ results = auto_feedback_icl.predict(completion_id=completion_id)
+ console.print_json(results)
+ return
+
+ if file:
+ with open(file, "r") as f:
+ content = f.read()
+ results = auto_feedback_icl.predict(text=content)
+ console.print_json(results)
+
+
+@click.command()
+@click.option("--completion-id", required=True, help="Completion ID")
+def get_autofeedback_cli(completion_id: str):
+ """
+ Get an auto feedback by completion id
+ """
+ res = get_autofeedback(completion_id)
+ if res:
+ rich.print_json(json.dumps(res["data"], indent=4))

log10/feedback/cli/feedback.py

@@ -0,0 +1,129 @@
+import json
+
+import click
+from rich.console import Console
+from rich.table import Table
+from tqdm import tqdm
+
+from log10.feedback.feedback import Feedback, _get_feedback_list
+
+
+@click.command()
+@click.option("--task_id", prompt="Enter task id", help="Task ID")
+@click.option("--values", prompt="Enter task values", help="Feedback in JSON format")
+@click.option(
+ "--completion_tags_selector",
+ prompt="Enter completion tags selector",
+ help="Completion tags selector",
+)
+@click.option("--comment", help="Comment", default="")
+def create_feedback(task_id, values, completion_tags_selector, comment):
+ """
+ Add feedback to a group of completions associated with a task
+ """
+ click.echo("Creating feedback")
+ tags = completion_tags_selector.split(",")
+ values = json.loads(values)
+ feedback = Feedback().create(task_id=task_id, values=values, completion_tags_selector=tags, comment=comment)
+ click.echo(feedback.json())
+
+
+@click.command()
+@click.option(
+ "--offset", default=0, type=int, help="The starting index from which to begin the feedback fetch. Defaults to 0."
+)
+@click.option(
+ "--limit", default=25, type=int, help="The maximum number of feedback items to retrieve. Defaults to 25."
+)
+@click.option(
+ "--task_id",
+ default="",
+ type=str,
+ help="The specific Task ID to filter feedback. If not provided, feedback for all tasks will be fetched.",
+)
+def list_feedback(offset, limit, task_id):
+ """
+ List feedback based on the provided criteria. This command allows fetching feedback for a specific task or across all tasks,
+ with control over the starting point and the number of items to retrieve.
+ """
+ feedback_data = _get_feedback_list(offset, limit, task_id)
+ data_for_table = []
+ for feedback in feedback_data:
+ data_for_table.append(
+ {
+ "id": feedback["id"],
+ "task_name": feedback["task_name"],
+ "feedback": json.dumps(feedback["json_values"], ensure_ascii=False),
+ "matched_completion_ids": ",".join(feedback["matched_completion_ids"]),
+ }
+ )
+ table = Table(title="Feedback")
+ table.add_column("ID")
+ table.add_column("Task Name")
+ table.add_column("Feedback")
+ table.add_column("Completion ID")
+
+ for item in data_for_table:
+ table.add_row(item["id"], item["task_name"], item["feedback"], item["matched_completion_ids"])
+ console = Console()
+ console.print(table)
+ console.print(f"Total feedback: {len(feedback_data)}")
+
+
+@click.command()
+@click.option("--id", required=True, help="Get feedback by ID")
+def get_feedback(id):
+ """
+ Get feedback based on provided ID.
+ """
+ try:
+ res = Feedback().get(id)
+ except Exception as e:
+ click.echo(f"Error fetching feedback {e}")
+ if hasattr(e, "response") and hasattr(e.response, "json") and "error" in e.response.json():
+ click.echo(e.response.json()["error"])
+ return
+ console = Console()
+ feedback = json.dumps(res.json(), indent=4)
+ console.print_json(feedback)
+
+
+@click.command()
+@click.option(
+ "--offset",
+ default=0,
+ help="The starting index from which to begin the feedback fetch. Leave empty to start from the beginning.",
+)
+@click.option(
+ "--limit", default="", help="The maximum number of feedback items to retrieve. Leave empty to retrieve all."
+)
+@click.option(
+ "--task_id",
+ default="",
+ type=str,
+ help="The specific Task ID to filter feedback. If not provided, feedback for all tasks will be fetched.",
+)
+@click.option(
+ "--file",
+ "-f",
+ type=str,
+ required=False,
+ help="Path to the file where the feedback will be saved. The feedback data is saved in JSON Lines (jsonl) format. If not specified, feedback will be printed to stdout.",
+)
+def download_feedback(offset, limit, task_id, file):
+ """
+ Download feedback based on the provided criteria. This command allows fetching feedback for a specific task or across all tasks,
+ with control over the starting point and the number of items to retrieve.
+ """
+ feedback_data = _get_feedback_list(offset, limit, task_id)
+
+ console = Console()
+ if not file:
+ for feedback in feedback_data:
+ console.print_json(json.dumps(feedback, indent=4))
+ return
+
+ with open(file, "w") as f:
+ console.print(f"Saving feedback to {file}")
+ for feedback in tqdm(feedback_data):
+ f.write(json.dumps(feedback) + "\n")

log10/feedback/cli/feedback_task.py

@@ -0,0 +1,80 @@
+import json
+
+import click
+from rich.console import Console
+from rich.table import Table
+
+from log10._httpx_utils import _get_time_diff
+from log10.feedback.feedback_task import FeedbackTask
+
+
+# create a cli interface for FeebackTask.create function
+@click.command()
+@click.option("--name", prompt="Enter feedback task name", help="Name of the task")
+@click.option("--task_schema", prompt="Enter feedback task schema", help="Task schema")
+@click.option("--instruction", help="Task instruction", default="")
+@click.option(
+ "--completion_tags_selector",
+ help="Completion tags selector",
+)
+def create_feedback_task(name, task_schema, instruction, completion_tags_selector=None):
+ click.echo("Creating feedback task")
+ tags = []
+
+ if completion_tags_selector:
+ tags = completion_tags_selector.split(",")
+
+ task_schema = json.loads(task_schema)
+ task = FeedbackTask().create(
+ name=name, task_schema=task_schema, completion_tags_selector=tags, instruction=instruction
+ )
+ click.echo(f"Use this task_id to add feedback: {task.json()['id']}")
+
+
+@click.command()
+@click.option("--limit", default=25, help="Number of feedback tasks to fetch")
+@click.option("--offset", default=0, help="Offset for the feedback tasks")
+def list_feedback_task(limit, offset):
+ res = FeedbackTask().list(limit=limit, offset=offset)
+ feedback_tasks = res.json()
+
+ data_for_table = []
+
+ for task in feedback_tasks["data"]:
+ data_for_table.append(
+ {
+ "id": task["id"],
+ "created_at": _get_time_diff(task["created_at"]),
+ "name": task["name"],
+ "required": task["json_schema"]["required"],
+ "instruction": task["instruction"],
+ }
+ )
+
+ table = Table(title="Feedback Tasks")
+ table.add_column("ID", style="dim")
+ table.add_column("Created At")
+ table.add_column("Name")
+ table.add_column("Required")
+ table.add_column("Instruction")
+ for item in data_for_table:
+ required = ", ".join(item["required"]) if item["required"] else ""
+ table.add_row(item["id"], item["created_at"], item["name"], required, item["instruction"])
+
+ console = Console()
+ console.print(table)
+
+
+@click.command()
+@click.option("--id", help="Get feedback task by ID")
+def get_feedback_task(id):
+ try:
+ res = FeedbackTask().get(id)
+ except Exception as e:
+ click.echo(f"Error fetching feedback task {e}")
+ if hasattr(e, "response") and hasattr(e.response, "json") and "error" in e.response.json():
+ click.echo(e.response.json()["error"])
+ return
+ task = json.dumps(res.json())
+ console = Console()
+ console.print_json(task)