ENG-834 Add fetching autofeedback by completion id to cli

log10/__main__.py

@@ -1,7 +1,7 @@
 import click
 
 from log10.completions.completions import benchmark_models, download_completions, get_completion, list_completions
-from log10.feedback.autofeedback import auto_feedback_icl
+from log10.feedback.autofeedback import auto_feedback_icl, get_autofeedback
 from log10.feedback.feedback import create_feedback, download_feedback, get_feedback, list_feedback
 from log10.feedback.feedback_task import create_feedback_task, get_feedback_task, list_feedback_task
 
@@ -35,6 +35,14 @@ def feedback_task():
  pass
 
 
+@click.group()
+def autofeedback():
+ """
+ Manage autofeedback for completions i.e. generated feedback from log10
+ """
+ pass
+
+
 cli.add_command(completions)
 completions.add_command(list_completions, "list")
 completions.add_command(get_completion, "get")
@@ -53,5 +61,8 @@ def feedback_task():
 feedback_task.add_command(list_feedback_task, "list")
 feedback_task.add_command(get_feedback_task, "get")
 
+cli.add_command(autofeedback)
+autofeedback.add_command(get_autofeedback, "get")
+
 if __name__ == "__main__":
  cli()

log10/_httpx_utils.py

@@ -85,6 +85,28 @@ def _try_post_request(url: str, payload: dict = {}) -> httpx.Response:
  logger.error(f"Failed to insert in log10: {payload} with error {err}")
 
 
+def _try_post_graphql_request(query: str, variables: dict = {}) -> httpx.Response:
+ url = "https://graphql.log10.io/graphql"
+ headers = {"content-type": "application/json", "x-api-token": _log10_config.token}
+
+ payload = {"query": query, "variables": variables}
+ res = None
+ try:
+ res = httpx_client.post(url, headers=headers, json=payload)
+ res.raise_for_status()
+ return res
+ except httpx.HTTPError as http_err:
+ if "401" in str(http_err):
+ logger.error(
+ "Failed anthorization. Please verify that LOG10_TOKEN and LOG10_ORG_ID are set correctly and try again."
+ + "\nSee https://github.com/log10-io/log10#%EF%B8%8F-setup for details"
+ )
+ else:
+ logger.error(f"Failed with error: {http_err}")
+ except Exception as err:
+ logger.error(f"Failed to make requests to log10 graphQL: {payload} with error {err}")
+
+
 async def _try_post_request_async(url: str, payload: dict = {}) -> httpx.Response:
  headers = {
  "x-log10-token": _log10_config.token,

log10/feedback/autofeedback.py

@@ -4,10 +4,14 @@
 from types import FunctionType
 
 import click
+import httpx
+import rich
 from rich.console import Console
 
+from log10._httpx_utils import _try_post_graphql_request
 from log10.completions.completions import _get_completion
 from log10.feedback.feedback import _get_feedback_list
+from log10.llm import Log10Config
 from log10.load import log10_session
 
 
@@ -22,6 +26,8 @@
 logger = logging.getLogger("LOG10")
 logger.setLevel(logging.INFO)
 
+_log10_config = Log10Config()
+
 
 class AutoFeedbackICL:
  """
@@ -95,6 +101,33 @@ def predict(self, text: str = None, completion_id: str = None) -> str:
  return ret
 
 
+def _get_autofeedback(completion_id: str) -> httpx.Response:
+ query = """
+ query OrganizationCompletion($orgId: String!, $id: String!) {
+ organization(id: $orgId) {
+ completion(id: $id) {
+ id
+ autoFeedback {
+ id
+ status
+ jsonValues
+ comment
+ }
+ }
+ }
+ }
+ """
+
+ variables = {"orgId": _log10_config.org_id, "id": completion_id}
+
+ response = _try_post_graphql_request(query, variables)
+
+ if response.status_code == 200:
+ return response.json()
+ else:
+ response.raise_for_status()
+
+
 @click.command()
 @click.option("--task_id", help="Feedback task ID")
 @click.option("--content", help="Completion content")
@@ -119,3 +152,14 @@ def auto_feedback_icl(task_id: str, content: str, file: str, completion_id: str,
  content = f.read()
  results = auto_feedback_icl.predict(text=content)
  console.print_json(results)
+
+
+@click.command()
+@click.option("--completion_id", prompt="Enter completion id", help="Completion ID")
+def get_autofeedback(completion_id: str):
+ """
+ Get a auto feedback by completion id
+ """
+ res = _get_autofeedback(completion_id)
+ rich.print_json(json.dumps(res["data"], indent=4))
+ return json.dumps(res)