From 113711a1dd4080e189d85501a6fa688647fca79a Mon Sep 17 00:00:00 2001 From: Senko Rasic Date: Mon, 10 Jun 2024 09:47:15 +0200 Subject: [PATCH] add telemetry for project import; small telemetry fixes --- core/agents/importer.py | 11 +++++++++++ core/cli/main.py | 3 --- core/llm/base.py | 5 ++++- core/telemetry/__init__.py | 1 + tests/llm/test_openai.py | 2 +- 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/core/agents/importer.py b/core/agents/importer.py index 4d6a1f521..00101ba0b 100644 --- a/core/agents/importer.py +++ b/core/agents/importer.py @@ -6,6 +6,7 @@ from core.db.models import Complexity from core.llm.parser import JSONParser from core.log import get_logger +from core.telemetry import telemetry from core.templates.example_project import EXAMPLE_PROJECT_DESCRIPTION log = get_logger(__name__) @@ -84,3 +85,13 @@ async def analyze_project(self): "complexity": Complexity.HARD if len(self.current_state.files) > 5 else Complexity.SIMPLE, } ] + + n_lines = sum(len(f.content.content.splitlines()) for f in self.current_state.files) + await telemetry.trace_code_event( + "existing-project", + { + "num_files": len(self.current_state.files), + "num_lines": n_lines, + "description": llm_response, + }, + ) diff --git a/core/cli/main.py b/core/cli/main.py index d7f66ad69..b591a6c51 100644 --- a/core/cli/main.py +++ b/core/cli/main.py @@ -28,7 +28,6 @@ async def run_project(sm: StateManager, ui: UIBase) -> bool: :return: True if the orchestrator exited successfully, False otherwise. """ - telemetry.start() telemetry.set("app_id", str(sm.project.id)) telemetry.set("initial_prompt", sm.current_state.specification.description) @@ -58,7 +57,6 @@ async def run_project(sm: StateManager, ui: UIBase) -> bool: source=pythagora_source, ) - await telemetry.send() return success @@ -147,7 +145,6 @@ async def run_pythagora_session(sm: StateManager, ui: UIBase, args: Namespace): if args.project or args.branch or args.step: telemetry.set("is_continuation", True) - # FIXME: we should send the project stage and other runtime info to the UI success = await load_project(sm, args.project, args.branch, args.step) if not success: return False diff --git a/core/llm/base.py b/core/llm/base.py index afe7f3219..afc7ff67f 100644 --- a/core/llm/base.py +++ b/core/llm/base.py @@ -181,6 +181,7 @@ async def __call__( remaining_retries -= 1 request_log.messages = convo.messages[:] request_log.response = None + request_log.status = LLMRequestStatus.SUCCESS request_log.error = None response = None @@ -278,7 +279,9 @@ async def __call__( response = parser(response) break except ValueError as err: - log.debug(f"Error parsing GPT response: {err}, asking LLM to retry", exc_info=True) + request_log.error = f"Error parsing response: {err}" + request_log.status = LLMRequestStatus.ERROR + log.debug(f"Error parsing LLM response: {err}, asking LLM to retry", exc_info=True) convo.assistant(response) convo.user(f"Error parsing response: {err}. Please output your response EXACTLY as requested.") continue diff --git a/core/telemetry/__init__.py b/core/telemetry/__init__.py index 18725babc..db10dbc94 100644 --- a/core/telemetry/__init__.py +++ b/core/telemetry/__init__.py @@ -321,6 +321,7 @@ async def send(self, event: str = "pilot-telemetry"): Note: this method clears all telemetry data after sending it. """ if not self.enabled: + log.debug("Telemetry.send(): telemetry is disabled, not sending data") return if self.endpoint is None: diff --git a/tests/llm/test_openai.py b/tests/llm/test_openai.py index a17b83a40..4a42fb01e 100644 --- a/tests/llm/test_openai.py +++ b/tests/llm/test_openai.py @@ -114,7 +114,7 @@ async def test_openai_parser_fails(mock_AsyncOpenAI): llm = OpenAIClient(cfg) - with pytest.raises(APIError, match="Error parsing LLM response"): + with pytest.raises(APIError, match="Error parsing response"): await llm(convo, parser=parser, max_retries=1)