[feat] Add codecov client

src/integrations/codecov/codecov_client.py

@@ -0,0 +1,37 @@
+import requests
+
+
+CODECOV_TOKEN = 'FETCH FROM ENV'
+class CodecovClient:
+    @staticmethod
+    def fetch_coverage(owner_username, repo_name, pullid, token=CODECOV_TOKEN):
+        url = f"https://api.codecov.io/api/v2/github/{owner_username}/repos/{repo_name}/pulls/{pullid}"
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "Accept": "application/json"
+        }
+        response = requests.get(url, headers=headers)
+        if response.status_code == 200:
+            return response.json()
+        else:
+            response.raise_for_status()
+
+    @staticmethod
+    def fetch_test_results_for_commit(owner_username, repo_name, latest_commit_sha, token=CODECOV_TOKEN):
+        url = f"https://api.codecov.io/api/v2/github/{owner_username}/repos/{repo_name}/test-results"
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "Accept": "application/json",
+            latest_commit_sha: latest_commit_sha,
+        }
+        response = requests.get(url, headers=headers)
+        if response.status_code == 200:
+            return response.json()
+        else:
+            response.raise_for_status()
+
+
+    @staticmethod
+    def ping():
+        return "pong"
+

src/seer/app.py

@@ -2,6 +2,7 @@
 import time
 
 import flask
+from integrations.codecov.codecov_client import CodecovClient
 import sentry_sdk
 from flask import Blueprint, Flask, jsonify
 from sentry_sdk.integrations.flask import FlaskIntegration
@@ -228,6 +229,9 @@ def autofix_evaluation_start_endpoint(data: AutofixEvaluationRequest) -> Autofix
 def codegen_unit_tests_endpoint(data: CodegenUnitTestsRequest) -> CodegenUnitTestsResponse:
     return codegen_unittest(data)
 
+@blueprint.route("/codecov-test", methods=["GET"])
+def test_codecov_client():
+    return CodecovClient.ping()
 
 @json_api(blueprint, "/v1/automation/codegen/unit-tests/state")
 def codegen_unit_tests_state_endpoint(

src/seer/automation/agent/agent.py

@@ -63,7 +63,6 @@ def get_completion(self):
 
     def run_iteration(self, context: Optional[AutofixContext] = None):
         logger.debug(f"----[{self.name}] Running Iteration {self.iterations}----")
-
         message, usage = self.get_completion()
 
         self.memory.append(message)
@@ -127,7 +126,6 @@ def run(self, prompt: str, context: Optional[AutofixContext] = None):
             raise MaxIterationsReachedException(
                 f"Agent {self.name} reached maximum iterations without finishing."
             )
-
         return self.get_last_message_content()
 
     def reset_iterations(self):

src/seer/automation/agent/client.py

@@ -177,7 +177,6 @@ def completion(
             )
 
         claude_messages = self._format_messages_for_claude_input(messages)
-
         # ask Claude for a response
         params: dict[str, Any] = {
             "model": model,
@@ -198,7 +197,6 @@ def completion(
             prompt_tokens=completion.usage.input_tokens,
             total_tokens=completion.usage.input_tokens + completion.usage.output_tokens,
         )
-
         langfuse_context.update_current_observation(model=model, usage=usage)
 
         return message, usage

src/seer/automation/codebase/repo_client.py

@@ -49,7 +49,6 @@ def get_repo_app_permissions(
 @inject
 def get_github_token_auth(config: AppConfig = injected) -> Auth.Token | None:
     github_token = config.GITHUB_TOKEN
-
     if github_token is None:
         return None
 
@@ -449,5 +448,4 @@ def get_pr_diff_content(self, pr_url: str) -> str:
         data = requests.get(pr_url, headers=headers)
 
         data.raise_for_status()  # Raise an exception for HTTP errors
-
         return data.text

src/seer/automation/codegen/prompts.py

@@ -18,7 +18,7 @@ def format_system_msg():
         )
 
     @staticmethod
-    def format_plan_step_msg(diff_str: str):
+    def format_plan_step_msg(diff_str: str, has_coverage_info: bool | str = False, has_test_result_info: bool | str = False):
         return textwrap.dedent(
             """\
             You are given the below code changes as a diff:
@@ -42,6 +42,81 @@ def format_plan_step_msg(diff_str: str):
         ).format(
             diff_str=diff_str,
         )
+
+    @staticmethod
+    def format_additional_code_coverage_info(coverage_info_str: str):
+        return textwrap.dedent(
+            """\
+            You are given the following code coverage information for the current diff as a json object:
+            {coverage_info_str}
+
+            # Additional Instructions for Using Code Coverage Information:
+
+            1. Analyze the provided code coverage data carefully. Pay attention to:
+               - Lines that are marked as 'miss' or 'partial'
+               - The overall coverage percentage for each file
+               - Any significant differences between 'base_totals' and 'head_totals'
+
+            2. Prioritize creating tests for:
+               - Uncovered lines (marked as 'miss')
+               - Partially covered lines (marked as 'partial')
+               - Files with lower overall coverage percentages
+               - New or modified code that has resulted in coverage decreases
+
+            3. For each file in the diff:
+               - Compare the 'base_totals' and 'head_totals' to identify changes in coverage
+               - Focus on increasing both line and branch coverage
+               - Pay special attention to changes in complexity and ensure they are adequately tested
+
+            4. When designing tests:
+               - Aim to increase the 'hits' count while decreasing 'misses' and 'partials'
+               - Consider edge cases and boundary conditions, especially for partially covered lines
+               - Ensure that any increase in 'complexity_total' is matched with appropriate test coverage
+
+            Remember, the goal is not just to increase the coverage numbers, but to ensure that the new tests meaningfully verify the behavior of the code, especially focusing on the changes and additions in the current diff.
+
+            Integrate this information with the diff analysis to provide a comprehensive and targeted testing strategy.
+            """
+        ).format(
+            coverage_info_str=coverage_info_str,
+        )
+
+    @staticmethod
+    def format_additional_test_results_info(test_result_data: str):
+        return textwrap.dedent(
+            """\
+            You are provided with the following test result data for existing tests for the diff:
+            {test_result_data}
+
+            # Instructions for Analyzing Test Result Data:
+
+            1. Review the test result data for each existing test, focusing on:
+               - Failure messages
+               - Test duration
+               - Failure rates
+
+            2. For tests with failures:
+               - Analyze the failure messages to identify issues introduced or exposed by this commit
+               - Consider creating additional tests to isolate and address these failures
+               - Suggest improvements or refactoring for the existing tests if the failures seem to be related to the changes in this commit
+
+            3. For tests with unusually long durations:
+               - Evaluate if the commit has introduced performance issues
+               - Consider adding performance-related tests if long durations are unexpected and related to the changes
+
+            4. Use the 'name' field of each test to understand its purpose and coverage area:
+               - Identify gaps in test coverage based on the names and purposes of existing tests, especially in relation to the changes in this commit
+               - Suggest new tests that complement the existing test suite and specifically address the changes in this commit
+
+            5. Pay special attention to the 'failure_rate' for each test:
+               - For tests with high failure rates, investigate if the failures are directly related to the changes in this commit
+               - For tests that previously passed but now fail, focus on understanding what in the commit might have caused this change
+
+            Use this information to enhance your test creation strategy, ensuring that new tests reinforce areas of failure, and improve overall test suite effectiveness in the context of the changes introduced.
+            """
+        ).format(
+            test_result_data=test_result_data,
+        )
 
     @staticmethod
     def format_find_unit_test_pattern_step_msg(diff_str: str):

src/seer/automation/codegen/tasks.py

@@ -20,15 +20,13 @@ def create_initial_unittest_run(request: CodegenUnitTestsRequest) -> DbState[Cod
         cur.status = CodegenStatus.PENDING
         cur.signals = []
         cur.mark_triggered()
-
     return state
 
 
 def codegen_unittest(request: CodegenUnitTestsRequest):
     state = create_initial_unittest_run(request)
 
     cur_state = state.get()
-
     # Process has no further work.
     # if cur_state.status in CodegenStatus.terminal():
     #     logger.warning(f"Ignoring job, state {cur_state.status}")

src/seer/automation/codegen/unit_test_coding_component.py

@@ -17,6 +17,7 @@
 from seer.automation.component import BaseComponent
 from seer.automation.models import FileChange
 from seer.automation.utils import escape_multi_xml, extract_text_inside_tags
+from integrations.codecov.codecov_client import CodecovClient
 
 logger = logging.getLogger(__name__)
 
@@ -39,7 +40,7 @@ def _get_plan(self, agent: LlmAgent, prompt: str) -> str:
     def _generate_tests(self, agent: LlmAgent, prompt: str) -> str:
         return agent.run(prompt=prompt)
 
-    def invoke(self, request: CodeUnitTestRequest) -> CodeUnitTestOutput | None:
+    def invoke(self, request: CodeUnitTestRequest, codecov_client_params: dict | None = None) -> CodeUnitTestOutput | None:
         langfuse_context.update_current_trace(user_id="ram")
         tools = BaseTools(self.context)
 
@@ -50,6 +51,22 @@ def invoke(self, request: CodeUnitTestRequest) -> CodeUnitTestOutput | None:
             ),
         )
 
+        code_coverage_data = CodecovClient.fetch_coverage(
+            repo_name=codecov_client_params["repo_name"],
+            pullid=codecov_client_params["pullid"],
+            owner_username=codecov_client_params["owner_username"]
+        )
+
+        test_result_data = CodecovClient.fetch_test_results_for_commit(
+            repo_name=codecov_client_params["repo_name"],
+            pullid=codecov_client_params["pullid"],
+            owner_username=codecov_client_params["owner_username"],
+            latest_commit_sha="SHA GOES HERE"
+        )
+
+        print(code_coverage_data, test_result_data)
+        # Pass this into format_plan_step_msg if they exist. Then combine the prompts
+
         existing_test_design_response = self._get_test_design_summary(
             agent=agent,
             prompt=CodingUnitTestPrompts.format_find_unit_test_pattern_step_msg(
@@ -58,7 +75,9 @@ def invoke(self, request: CodeUnitTestRequest) -> CodeUnitTestOutput | None:
         )
 
         self._get_plan(
-            agent=agent, prompt=CodingUnitTestPrompts.format_plan_step_msg(diff_str=request.diff)
+            agent=agent, prompt=CodingUnitTestPrompts.format_plan_step_msg(
+                diff_str=request.diff
+            )
         )
 
         final_response = self._generate_tests(

src/seer/automation/codegen/unittest_step.py

@@ -53,13 +53,19 @@ def _invoke(self, **kwargs):
 
         repo_client = self.context.get_repo_client()
         pr = repo_client.repo.get_pull(self.request.pr_id)
+        codecov_client_params = {
+            "repo_name": self.request.repo_definition.name,
+            "pullid": self.request.pr_id,
+            "owner_username": self.request.repo_definition.owner
+        }
 
         diff_content = repo_client.get_pr_diff_content(pr.url)
 
         unittest_output = UnitTestCodingComponent(self.context).invoke(
             CodeUnitTestRequest(
                 diff=diff_content,
-            )
+            ),
+            codecov_client_params=codecov_client_params,
         )
 
         if unittest_output: