ENG-834 Add fetching autofeedback by completion id to cli (#175)

* Add fetching autofeedback by completion id to cli

* Updates based on feedback

* Address feedback on description and adding async call

cli_docs.md

@@ -194,6 +194,8 @@ And to download to a JSONL file, use `log10 feedback download --task_id 04405cbc
 
 To leverage your feedback and use AI to generate more feedback automatically, use [`log10 feedback predict`](#log10-feedback-predict). Please refer to this [doc](https://log10.io/docs/feedback/auto_feedback) for more details.
 
+To get auto generated feedback for a completion, use [`log10 feedback autofeedback get`](#log10-feedback-autofeedback-get)
+
 ## CLI References
 
 ### Completions
@@ -399,6 +401,8 @@ Options:
 $ log10 feedback predict --help
 Usage: log10 feedback predict [OPTIONS]
 
+ Generate feedback with existing human feedback based on in context learning
+
 Options:
  --task_id TEXT Feedback task ID
  --content TEXT Completion content
@@ -407,6 +411,17 @@ Options:
  --num_samples INTEGER Number of samples to use for few-shot learning
 ```
 
+#### log10 feedback autofeedback get
+```bash
+$ log10 feedback autofeedback get --help
+Usage: log10 feedback autofeedback get [OPTIONS]
+
+ Get an auto feedback by completion id
+
+Options:
+ --completion-id TEXT Completion ID [required]
+```
+
 ### Feedback Task
 
 ```bash

log10/__main__.py

@@ -1,7 +1,7 @@
 import click
 
 from log10.completions.completions import benchmark_models, download_completions, get_completion, list_completions
-from log10.feedback.autofeedback import auto_feedback_icl
+from log10.feedback.autofeedback import auto_feedback_icl, get_autofeedback_cli
 from log10.feedback.feedback import create_feedback, download_feedback, get_feedback, list_feedback
 from log10.feedback.feedback_task import create_feedback_task, get_feedback_task, list_feedback_task
 
@@ -19,14 +19,22 @@ def completions():
  pass
 
 
-@click.group()
+@click.group(name="feedback")
 def feedback():
  """
  Manage feedback for completions i.e. capturing feedback from users
  """
  pass
 
 
+@click.group(name="auto_feedback")
+def auto_feedback():
+ """
+ Manage auto feedback for completions i.e. capturing feedback from users
+ """
+ pass
+
+
 @click.group()
 def feedback_task():
  """
@@ -47,6 +55,9 @@ def feedback_task():
 feedback.add_command(get_feedback, "get")
 feedback.add_command(download_feedback, "download")
 feedback.add_command(auto_feedback_icl, "predict")
+feedback.add_command(auto_feedback, "autofeedback")
+# Subcommands for auto_feedback under feedback command
+auto_feedback.add_command(get_autofeedback_cli, "get")
 
 cli.add_command(feedback_task)
 feedback_task.add_command(create_feedback_task, "create")

log10/_httpx_utils.py

@@ -14,6 +14,7 @@
 
 logger: logging.Logger = logging.getLogger("LOG10")
 
+GRAPHQL_URL = "https://graphql.log10.io/graphql"
 
 _log10_config = Log10Config()
 base_url = _log10_config.url
@@ -85,6 +86,48 @@ def _try_post_request(url: str, payload: dict = {}) -> httpx.Response:
  logger.error(f"Failed to insert in log10: {payload} with error {err}")
 
 
+def _try_post_graphql_request(query: str, variables: dict = {}) -> httpx.Response:
+ headers = {"content-type": "application/json", "x-api-token": _log10_config.token}
+
+ payload = {"query": query, "variables": variables}
+ res = None
+ try:
+ res = httpx_client.post(GRAPHQL_URL, headers=headers, json=payload)
+ res.raise_for_status()
+ return res
+ except httpx.HTTPError as http_err:
+ if "401" in str(http_err):
+ logger.error(
+ "Failed anthorization. Please verify that LOG10_TOKEN and LOG10_ORG_ID are set correctly and try again."
+ + "\nSee https://github.com/log10-io/log10#%EF%B8%8F-setup for details"
+ )
+ else:
+ logger.error(f"Failed with error: {http_err}")
+ except Exception as err:
+ logger.error(f"Failed to make requests to log10 graphQL: {payload} with error {err}")
+
+
+async def _try_post_graphql_request_async(query: str, variables: dict = {}) -> httpx.Response:
+ headers = {"content-type": "application/json", "x-api-token": _log10_config.token}
+
+ payload = {"query": query, "variables": variables}
+ res = None
+ try:
+ res = await httpx_async_client.post(GRAPHQL_URL, headers=headers, json=payload)
+ res.raise_for_status()
+ return res
+ except httpx.HTTPError as http_err:
+ if "401" in str(http_err):
+ logger.error(
+ "Failed anthorization. Please verify that LOG10_TOKEN and LOG10_ORG_ID are set correctly and try again."
+ + "\nSee https://github.com/log10-io/log10#%EF%B8%8F-setup for details"
+ )
+ else:
+ logger.error(f"Failed with error: {http_err}")
+ except Exception as err:
+ logger.error(f"Failed to make requests to log10 graphQL: {payload} with error {err}")
+
+
 async def _try_post_request_async(url: str, payload: dict = {}) -> httpx.Response:
  headers = {
  "x-log10-token": _log10_config.token,

log10/feedback/autofeedback.py

@@ -4,10 +4,14 @@
 from types import FunctionType
 
 import click
+import httpx
+import rich
 from rich.console import Console
 
+from log10._httpx_utils import _try_post_graphql_request
 from log10.completions.completions import _get_completion
 from log10.feedback.feedback import _get_feedback_list
+from log10.llm import Log10Config
 from log10.load import log10_session
 
 
@@ -22,6 +26,8 @@
 logger = logging.getLogger("LOG10")
 logger.setLevel(logging.INFO)
 
+_log10_config = Log10Config()
+
 
 class AutoFeedbackICL:
  """
@@ -95,13 +101,47 @@ def predict(self, text: str = None, completion_id: str = None) -> str:
  return ret
 
 
+def get_autofeedback(completion_id: str) -> httpx.Response:
+ query = """
+ query OrganizationCompletion($orgId: String!, $id: String!) {
+ organization(id: $orgId) {
+ completion(id: $id) {
+ id
+ autoFeedback {
+ id
+ status
+ jsonValues
+ comment
+ }
+ }
+ }
+ }
+ """
+
+ variables = {"orgId": _log10_config.org_id, "id": completion_id}
+
+ response = _try_post_graphql_request(query, variables)
+
+ if response is None:
+ logger.error(f"Failed to get auto feedback for completion {completion_id}")
+ return None
+
+ if response.status_code == 200:
+ return response.json()
+ else:
+ response.raise_for_status()
+
+
 @click.command()
 @click.option("--task_id", help="Feedback task ID")
 @click.option("--content", help="Completion content")
 @click.option("--file", "-f", help="File containing completion content")
 @click.option("--completion_id", help="Completion ID")
 @click.option("--num_samples", default=5, help="Number of samples to use for few-shot learning")
 def auto_feedback_icl(task_id: str, content: str, file: str, completion_id: str, num_samples: int):
+ """
+ Generate feedback with existing human feedback based on in context learning
+ """
  options_count = sum([1 for option in [content, file, completion_id] if option])
  if options_count > 1:
  click.echo("Only one of --content, --file, or --completion_id should be provided.")
@@ -119,3 +159,14 @@ def auto_feedback_icl(task_id: str, content: str, file: str, completion_id: str,
  content = f.read()
  results = auto_feedback_icl.predict(text=content)
  console.print_json(results)
+
+
+@click.command()
+@click.option("--completion-id", required=True, help="Completion ID")
+def get_autofeedback_cli(completion_id: str):
+ """
+ Get an auto feedback by completion id
+ """
+ res = get_autofeedback(completion_id)
+ if res:
+ rich.print_json(json.dumps(res["data"], indent=4))

log10/feedback/feedback.py

@@ -181,7 +181,7 @@ def list_feedback(offset, limit, task_id):
 
 
 @click.command()
-@click.option("--id", help="Get feedback by ID")
+@click.option("--id", required=True, help="Get feedback by ID")
 def get_feedback(id):
  """
  Get feedback based on provided ID.