[draft][wip] Add codecov client

src/integrations/codecov/codecov_client.py

@@ -0,0 +1,37 @@
+import requests
+
+
+CODECOV_TOKEN = "FETCH FROM ENV"
+
+
+class CodecovClient:
+ @staticmethod
+ def fetch_coverage(owner_username, repo_name, pullid, token=CODECOV_TOKEN):
+ url = f"https://api.codecov.io/api/v2/github/{owner_username}/repos/{repo_name}/pulls/{pullid}"
+ headers = {"Authorization": f"Bearer {token}", "Accept": "application/json"}
+ response = requests.get(url, headers=headers)
+ if response.status_code == 200:
+ return response.text
+ else:
+ return None
+
+ @staticmethod
+ def fetch_test_results_for_commit(
+ owner_username, repo_name, latest_commit_sha, token=CODECOV_TOKEN
+ ):
+ url = f"https://api.codecov.io/api/v2/github/{owner_username}/repos/{repo_name}/test-results?commit_id={latest_commit_sha}&outcome=failure"
+ headers = {
+ "Authorization": f"Bearer {token}",
+ "Accept": "application/json",
+ }
+ response = requests.get(url, headers=headers)
+ if response.status_code == 200:
+ if response.json()["count"] == 0:
+ return None
+ return response.text
+ else:
+ return None
+
+ @staticmethod
+ def ping():
+ return "pong"

src/seer/app.py

@@ -2,6 +2,7 @@
 import time
 
 import flask
+from integrations.codecov import CodecovClient
 import sentry_sdk
 from flask import Blueprint, Flask, jsonify
 from sentry_sdk.integrations.flask import FlaskIntegration
@@ -229,6 +230,11 @@ def codegen_unit_tests_endpoint(data: CodegenUnitTestsRequest) -> CodegenUnitTes
  return codegen_unittest(data)
 
 
+@blueprint.route("/codecov-test", methods=["GET"])
+def test_codecov_client():
+ return CodecovClient.ping()
+
+
 @json_api(blueprint, "/v1/automation/codegen/unit-tests/state")
 def codegen_unit_tests_state_endpoint(
  data: CodegenUnitTestsStateRequest,

src/seer/automation/codebase/repo_client.py

@@ -49,7 +49,6 @@ def get_repo_app_permissions(
 @inject
 def get_github_token_auth(config: AppConfig = injected) -> Auth.Token | None:
  github_token = config.GITHUB_TOKEN
-
  if github_token is None:
  return None
 
@@ -62,7 +61,6 @@ def get_write_app_credentials(config: AppConfig = injected) -> tuple[int | str |
  private_key = config.GITHUB_PRIVATE_KEY
 
  if not app_id or not private_key:
-
  return None, None
 
  return app_id, private_key
@@ -449,5 +447,15 @@ def get_pr_diff_content(self, pr_url: str) -> str:
  data = requests.get(pr_url, headers=headers)
 
  data.raise_for_status() # Raise an exception for HTTP errors
-
  return data.text
+
+ def get_pr_head_sha(self, pr_url: str) -> str:
+ requester = self.repo._requester
+ headers = {
+ "Authorization": f"{requester.auth.token_type} {requester.auth.token}", # type: ignore
+ "Accept": "application/vnd.github.raw+json",
+ }
+
+ data = requests.get(pr_url, headers=headers)
+ data.raise_for_status() # Raise an exception for HTTP errors
+ return data.json()["head"]["sha"]

src/seer/automation/codegen/prompts.py

@@ -18,8 +18,12 @@ def format_system_msg():
  )
 
  @staticmethod
- def format_plan_step_msg(diff_str: str):
- return textwrap.dedent(
+ def format_plan_step_msg(
+ diff_str: str,
+ has_coverage_info: str | None = None,
+ has_test_result_info: str | None = None,
+ ):
+ base_msg = textwrap.dedent(
  """\
  You are given the below code changes as a diff:
  {diff_str}
@@ -34,14 +38,37 @@ def format_plan_step_msg(diff_str: str):
  # Guidelines:
  - No placeholders are allowed, the unit test must be clear and detailed.
  - Make sure you use the tools provided to look through the codebase and at the files you are changing before outputting your suggested fix.
- - The unit tests must be comprehensive. Do not provide temporary examples, placeholders or incomplete ones.
+ - The unit tests must be comprehensive. Do not provide temporary examples, placeholders, or incomplete ones.
  - In your suggested unit tests, whenever you are providing code, provide explicit diffs to show the exact changes that need to be made.
  - All your changes should be in test files.
  - EVERY TIME before you use a tool, think step-by-step each time before using the tools provided to you.
  - You also MUST think step-by-step before giving the final answer."""
- ).format(
- diff_str=diff_str,
- )
+ ).format(diff_str=diff_str)
+
+ if has_coverage_info:
+ coverage_info_msg = textwrap.dedent(
+ """\
+ You are also given the following code coverage information for the current diff as a JSON object:
+ {coverage_info_str}
+
+ Remember, the goal is not just to improve coverage numbers but to verify the behavior of the code meaningfully, focusing on the recent changes.
+ Integrate this information with your diff analysis to provide a comprehensive and targeted testing strategy.
+ """
+ ).format(coverage_info_str=has_coverage_info)
+ base_msg += "\n\n" + coverage_info_msg
+
+ if has_test_result_info:
+ test_result_info_msg = textwrap.dedent(
+ """\
+ You are provided with the following test result data for existing tests related to the diff:
+ {test_result_data}
+
+ Use this information to enhance your test creation strategy, ensuring new tests reinforce areas of failure and improve overall test suite effectiveness in the context of the introduced changes.
+ """
+ ).format(test_result_data=has_test_result_info)
+ base_msg += "\n\n" + test_result_info_msg
+
+ return base_msg
 
  @staticmethod
  def format_find_unit_test_pattern_step_msg(diff_str: str):

src/seer/automation/codegen/unit_test_coding_component.py

@@ -17,6 +17,7 @@
 from seer.automation.component import BaseComponent
 from seer.automation.models import FileChange
 from seer.automation.utils import escape_multi_xml, extract_text_inside_tags
+from integrations.codecov import CodecovClient
 
 logger = logging.getLogger(__name__)
 
@@ -39,7 +40,9 @@ def _get_plan(self, agent: LlmAgent, prompt: str) -> str:
  def _generate_tests(self, agent: LlmAgent, prompt: str) -> str:
  return agent.run(prompt=prompt)
 
- def invoke(self, request: CodeUnitTestRequest) -> CodeUnitTestOutput | None:
+ def invoke(
+ self, request: CodeUnitTestRequest, codecov_client_params: dict | None = None
+ ) -> CodeUnitTestOutput | None:
  langfuse_context.update_current_trace(user_id="ram")
  tools = BaseTools(self.context)
 
@@ -50,6 +53,20 @@ def invoke(self, request: CodeUnitTestRequest) -> CodeUnitTestOutput | None:
  ),
  )
 
+ code_coverage_data = CodecovClient.fetch_coverage(
+ repo_name=codecov_client_params["repo_name"],
+ pullid=codecov_client_params["pullid"],
+ owner_username=codecov_client_params["owner_username"],
+ )
+
+ test_result_data = CodecovClient.fetch_test_results_for_commit(
+ repo_name=codecov_client_params["repo_name"],
+ owner_username=codecov_client_params["owner_username"],
+ latest_commit_sha=codecov_client_params["head_sha"],
+ )
+
+ # Pass this into format_plan_step_msg if they exist. Then combine the prompts
+
  existing_test_design_response = self._get_test_design_summary(
  agent=agent,
  prompt=CodingUnitTestPrompts.format_find_unit_test_pattern_step_msg(
@@ -58,7 +75,12 @@ def invoke(self, request: CodeUnitTestRequest) -> CodeUnitTestOutput | None:
  )
 
  self._get_plan(
- agent=agent, prompt=CodingUnitTestPrompts.format_plan_step_msg(diff_str=request.diff)
+ agent=agent,
+ prompt=CodingUnitTestPrompts.format_plan_step_msg(
+ diff_str=request.diff,
+ has_coverage_info=code_coverage_data,
+ has_test_result_info=test_result_data,
+ ),
  )
 
  final_response = self._generate_tests(
@@ -70,7 +92,6 @@ def invoke(self, request: CodeUnitTestRequest) -> CodeUnitTestOutput | None:
 
  if not final_response:
  return None
-
  plan_steps_content = extract_text_inside_tags(final_response, "plan_steps")
 
  if len(plan_steps_content) == 0:

src/seer/automation/codegen/unittest_step.py

@@ -53,13 +53,22 @@ def _invoke(self, **kwargs):
 
  repo_client = self.context.get_repo_client()
  pr = repo_client.repo.get_pull(self.request.pr_id)
-
  diff_content = repo_client.get_pr_diff_content(pr.url)
 
+ latest_commit_sha = repo_client.get_pr_head_sha(pr.url)
+
+ codecov_client_params = {
+ "repo_name": self.request.repo_definition.name,
+ "pullid": self.request.pr_id,
+ "owner_username": self.request.repo_definition.owner,
+ "head_sha": latest_commit_sha,
+ }
+
  unittest_output = UnitTestCodingComponent(self.context).invoke(
  CodeUnitTestRequest(
  diff=diff_content,
- )
+ ),
+ codecov_client_params=codecov_client_params,
  )
 
  if unittest_output: