ENG-879 ENG-882 Move cli logic into its own folder (#198)

* Print out missing cli modules

* Move cli code into its own folder

* Move completions cli to its own file

* Add e2e tests for cli commands

* Add support cli to display tool_calls

* Update poetry.lock file

* Rename cli extra group dependencies from 'cliModules' to 'cli'

* Add checking pandas, rich, tabulate and simplified if conditions

* Separate llm vs cli tests

.github/workflows/test.yml

@@ -63,7 +63,10 @@ jobs:
       - name: Install dependencies
         run: poetry install --all-extras
 
-      - name: Run dispatch tests
+      - name: Run cli tests
+        run: poetry run pytest -vv tests/test_cli.py
+
+      - name: Run dispatch llm tests
         if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request' }}
         run: |
           echo "This is a dispatch event"
@@ -113,13 +116,13 @@ jobs:
 
           if $empty_inputs; then
             echo "All variables are empty"
-            poetry run pytest -vv tests/
+            poetry run pytest -vv tests/ --ignore=tests/test_cli.py
             poetry run pytest --llm_provider=anthropic -vv tests/test_magentic.py
           fi
 
-      - name: Run scheduled tests
+      - name: Run scheduled llm tests
         if: ${{ github.event_name == 'schedule' }}
         run: |
           echo "This is a schedule event"
-          poetry run pytest -vv tests/
+          poetry run pytest -vv tests/ --ignore=tests/test_cli.py
           poetry run pytest --openai_model=gpt-4o -m chat -vv tests/test_openai.py

log10/__main__.py

@@ -1,68 +1,5 @@
-import click
+from log10.cli.cli_commands import cli
 
-from log10.completions.completions import benchmark_models, download_completions, get_completion, list_completions
-from log10.feedback.autofeedback import auto_feedback_icl, get_autofeedback_cli
-from log10.feedback.feedback import create_feedback, download_feedback, get_feedback, list_feedback
-from log10.feedback.feedback_task import create_feedback_task, get_feedback_task, list_feedback_task
-
-
-@click.group()
-def cli():
-    pass
-
-
-@click.group()
-def completions():
-    """
-    Manage logs from completions i.e. logs from users
-    """
-    pass
-
-
-@click.group(name="feedback")
-def feedback():
-    """
-    Manage feedback for completions i.e. capturing feedback from users
-    """
-    pass
-
-
-@click.group(name="auto_feedback")
-def auto_feedback():
-    """
-    Manage auto feedback for completions i.e. capturing feedback from users
-    """
-    pass
-
-
-@click.group()
-def feedback_task():
-    """
-    Manage tasks for feedback i.e. instructions and schema for feedback
-    """
-    pass
-
-
-cli.add_command(completions)
-completions.add_command(list_completions, "list")
-completions.add_command(get_completion, "get")
-completions.add_command(download_completions, "download")
-completions.add_command(benchmark_models, "benchmark_models")
-
-cli.add_command(feedback)
-feedback.add_command(create_feedback, "create")
-feedback.add_command(list_feedback, "list")
-feedback.add_command(get_feedback, "get")
-feedback.add_command(download_feedback, "download")
-feedback.add_command(auto_feedback_icl, "predict")
-feedback.add_command(auto_feedback, "autofeedback")
-# Subcommands for auto_feedback under feedback command
-auto_feedback.add_command(get_autofeedback_cli, "get")
-
-cli.add_command(feedback_task)
-feedback_task.add_command(create_feedback_task, "create")
-feedback_task.add_command(list_feedback_task, "list")
-feedback_task.add_command(get_feedback_task, "get")
 
 if __name__ == "__main__":
     cli()

log10/cli/autofeedback.py

@@ -0,0 +1,47 @@
+import json
+
+import click
+import rich
+from rich.console import Console
+
+from log10.feedback.autofeedback import AutoFeedbackICL, get_autofeedback
+
+
+@click.command()
+@click.option("--task_id", help="Feedback task ID")
+@click.option("--content", help="Completion content")
+@click.option("--file", "-f", help="File containing completion content")
+@click.option("--completion_id", help="Completion ID")
+@click.option("--num_samples", default=5, help="Number of samples to use for few-shot learning")
+def auto_feedback_icl(task_id: str, content: str, file: str, completion_id: str, num_samples: int):
+    """
+    Generate feedback with existing human feedback based on in context learning
+    """
+    options_count = sum([1 for option in [content, file, completion_id] if option])
+    if options_count > 1:
+        click.echo("Only one of --content, --file, or --completion_id should be provided.")
+        return
+
+    console = Console()
+    auto_feedback_icl = AutoFeedbackICL(task_id, num_samples=num_samples)
+    if completion_id:
+        results = auto_feedback_icl.predict(completion_id=completion_id)
+        console.print_json(results)
+        return
+
+    if file:
+        with open(file, "r") as f:
+            content = f.read()
+    results = auto_feedback_icl.predict(text=content)
+    console.print_json(results)
+
+
+@click.command()
+@click.option("--completion-id", required=True, help="Completion ID")
+def get_autofeedback_cli(completion_id: str):
+    """
+    Get an auto feedback by completion id
+    """
+    res = get_autofeedback(completion_id)
+    if res:
+        rich.print_json(json.dumps(res["data"], indent=4))

log10/cli/cli_commands.py

@@ -0,0 +1,74 @@
+try:
+    import click
+    import pandas  # noqa: F401
+    import rich  # noqa: F401
+    import tabulate  # noqa: F401
+except ImportError:
+    print(
+        "To use log10 cli you must install optional modules. Please install them with `pip install 'log10-io[cli]'`."
+    )
+    exit(1)
+
+from log10.cli.autofeedback import auto_feedback_icl, get_autofeedback_cli
+from log10.cli.completions import benchmark_models, download_completions, get_completion, list_completions
+from log10.cli.feedback import create_feedback, download_feedback, get_feedback, list_feedback
+from log10.cli.feedback_task import create_feedback_task, get_feedback_task, list_feedback_task
+
+
+@click.group()
+def cli():
+    pass
+
+
+@click.group()
+def completions():
+    """
+    Manage logs from completions i.e. logs from users
+    """
+    pass
+
+
+@click.group(name="feedback")
+def feedback():
+    """
+    Manage feedback for completions i.e. capturing feedback from users
+    """
+    pass
+
+
+@click.group(name="auto_feedback")
+def auto_feedback():
+    """
+    Manage auto feedback for completions i.e. capturing feedback from users
+    """
+    pass
+
+
+@click.group()
+def feedback_task():
+    """
+    Manage tasks for feedback i.e. instructions and schema for feedback
+    """
+    pass
+
+
+cli.add_command(completions)
+completions.add_command(list_completions, "list")
+completions.add_command(get_completion, "get")
+completions.add_command(download_completions, "download")
+completions.add_command(benchmark_models, "benchmark_models")
+
+cli.add_command(feedback)
+feedback.add_command(create_feedback, "create")
+feedback.add_command(list_feedback, "list")
+feedback.add_command(get_feedback, "get")
+feedback.add_command(download_feedback, "download")
+feedback.add_command(auto_feedback_icl, "predict")
+feedback.add_command(auto_feedback, "autofeedback")
+# Subcommands for auto_feedback under feedback command
+auto_feedback.add_command(get_autofeedback_cli, "get")
+
+cli.add_command(feedback_task)
+feedback_task.add_command(create_feedback_task, "create")
+feedback_task.add_command(list_feedback_task, "list")
+feedback_task.add_command(get_feedback_task, "get")