diff --git a/src/integrations/__init__.py b/src/integrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/integrations/codecov/ __init__.py b/src/integrations/codecov/ __init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/integrations/codecov/codecov_client.py b/src/integrations/codecov/codecov_client.py new file mode 100644 index 00000000..3e3a19b2 --- /dev/null +++ b/src/integrations/codecov/codecov_client.py @@ -0,0 +1,37 @@ +import requests + + +CODECOV_TOKEN = "FETCH FROM ENV" + + +class CodecovClient: + @staticmethod + def fetch_coverage(owner_username, repo_name, pullid, token=CODECOV_TOKEN): + url = f"https://api.codecov.io/api/v2/github/{owner_username}/repos/{repo_name}/pulls/{pullid}" + headers = {"Authorization": f"Bearer {token}", "Accept": "application/json"} + response = requests.get(url, headers=headers) + if response.status_code == 200: + return response.text + else: + return None + + @staticmethod + def fetch_test_results_for_commit( + owner_username, repo_name, latest_commit_sha, token=CODECOV_TOKEN + ): + url = f"https://api.codecov.io/api/v2/github/{owner_username}/repos/{repo_name}/test-results?commit_id={latest_commit_sha}&outcome=failure" + headers = { + "Authorization": f"Bearer {token}", + "Accept": "application/json", + } + response = requests.get(url, headers=headers) + if response.status_code == 200: + if response.json()["count"] == 0: + return None + return response.text + else: + return None + + @staticmethod + def ping(): + return "pong" diff --git a/src/seer/app.py b/src/seer/app.py index 62fdc5d6..8c61cd68 100644 --- a/src/seer/app.py +++ b/src/seer/app.py @@ -2,6 +2,7 @@ import time import flask +from integrations.codecov import CodecovClient import sentry_sdk from flask import Blueprint, Flask, jsonify from sentry_sdk.integrations.flask import FlaskIntegration @@ -229,6 +230,11 @@ def codegen_unit_tests_endpoint(data: CodegenUnitTestsRequest) -> CodegenUnitTes return codegen_unittest(data) +@blueprint.route("/codecov-test", methods=["GET"]) +def test_codecov_client(): + return CodecovClient.ping() + + @json_api(blueprint, "/v1/automation/codegen/unit-tests/state") def codegen_unit_tests_state_endpoint( data: CodegenUnitTestsStateRequest, diff --git a/src/seer/automation/codebase/repo_client.py b/src/seer/automation/codebase/repo_client.py index 59983381..1809d432 100644 --- a/src/seer/automation/codebase/repo_client.py +++ b/src/seer/automation/codebase/repo_client.py @@ -49,7 +49,6 @@ def get_repo_app_permissions( @inject def get_github_token_auth(config: AppConfig = injected) -> Auth.Token | None: github_token = config.GITHUB_TOKEN - if github_token is None: return None @@ -62,7 +61,6 @@ def get_write_app_credentials(config: AppConfig = injected) -> tuple[int | str | private_key = config.GITHUB_PRIVATE_KEY if not app_id or not private_key: - return None, None return app_id, private_key @@ -449,5 +447,15 @@ def get_pr_diff_content(self, pr_url: str) -> str: data = requests.get(pr_url, headers=headers) data.raise_for_status() # Raise an exception for HTTP errors - return data.text + + def get_pr_head_sha(self, pr_url: str) -> str: + requester = self.repo._requester + headers = { + "Authorization": f"{requester.auth.token_type} {requester.auth.token}", # type: ignore + "Accept": "application/vnd.github.raw+json", + } + + data = requests.get(pr_url, headers=headers) + data.raise_for_status() # Raise an exception for HTTP errors + return data.json()["head"]["sha"] diff --git a/src/seer/automation/codegen/prompts.py b/src/seer/automation/codegen/prompts.py index 234e11de..868f94c1 100644 --- a/src/seer/automation/codegen/prompts.py +++ b/src/seer/automation/codegen/prompts.py @@ -18,8 +18,12 @@ def format_system_msg(): ) @staticmethod - def format_plan_step_msg(diff_str: str): - return textwrap.dedent( + def format_plan_step_msg( + diff_str: str, + has_coverage_info: str | None = None, + has_test_result_info: str | None = None, + ): + base_msg = textwrap.dedent( """\ You are given the below code changes as a diff: {diff_str} @@ -34,14 +38,37 @@ def format_plan_step_msg(diff_str: str): # Guidelines: - No placeholders are allowed, the unit test must be clear and detailed. - Make sure you use the tools provided to look through the codebase and at the files you are changing before outputting your suggested fix. - - The unit tests must be comprehensive. Do not provide temporary examples, placeholders or incomplete ones. + - The unit tests must be comprehensive. Do not provide temporary examples, placeholders, or incomplete ones. - In your suggested unit tests, whenever you are providing code, provide explicit diffs to show the exact changes that need to be made. - All your changes should be in test files. - EVERY TIME before you use a tool, think step-by-step each time before using the tools provided to you. - You also MUST think step-by-step before giving the final answer.""" - ).format( - diff_str=diff_str, - ) + ).format(diff_str=diff_str) + + if has_coverage_info: + coverage_info_msg = textwrap.dedent( + """\ + You are also given the following code coverage information for the current diff as a JSON object: + {coverage_info_str} + + Remember, the goal is not just to improve coverage numbers but to verify the behavior of the code meaningfully, focusing on the recent changes. + Integrate this information with your diff analysis to provide a comprehensive and targeted testing strategy. + """ + ).format(coverage_info_str=has_coverage_info) + base_msg += "\n\n" + coverage_info_msg + + if has_test_result_info: + test_result_info_msg = textwrap.dedent( + """\ + You are provided with the following test result data for existing tests related to the diff: + {test_result_data} + + Use this information to enhance your test creation strategy, ensuring new tests reinforce areas of failure and improve overall test suite effectiveness in the context of the introduced changes. + """ + ).format(test_result_data=has_test_result_info) + base_msg += "\n\n" + test_result_info_msg + + return base_msg @staticmethod def format_find_unit_test_pattern_step_msg(diff_str: str): diff --git a/src/seer/automation/codegen/unit_test_coding_component.py b/src/seer/automation/codegen/unit_test_coding_component.py index ac7fbc52..7c482de8 100644 --- a/src/seer/automation/codegen/unit_test_coding_component.py +++ b/src/seer/automation/codegen/unit_test_coding_component.py @@ -17,6 +17,7 @@ from seer.automation.component import BaseComponent from seer.automation.models import FileChange from seer.automation.utils import escape_multi_xml, extract_text_inside_tags +from integrations.codecov import CodecovClient logger = logging.getLogger(__name__) @@ -39,7 +40,9 @@ def _get_plan(self, agent: LlmAgent, prompt: str) -> str: def _generate_tests(self, agent: LlmAgent, prompt: str) -> str: return agent.run(prompt=prompt) - def invoke(self, request: CodeUnitTestRequest) -> CodeUnitTestOutput | None: + def invoke( + self, request: CodeUnitTestRequest, codecov_client_params: dict | None = None + ) -> CodeUnitTestOutput | None: langfuse_context.update_current_trace(user_id="ram") tools = BaseTools(self.context) @@ -50,6 +53,20 @@ def invoke(self, request: CodeUnitTestRequest) -> CodeUnitTestOutput | None: ), ) + code_coverage_data = CodecovClient.fetch_coverage( + repo_name=codecov_client_params["repo_name"], + pullid=codecov_client_params["pullid"], + owner_username=codecov_client_params["owner_username"], + ) + + test_result_data = CodecovClient.fetch_test_results_for_commit( + repo_name=codecov_client_params["repo_name"], + owner_username=codecov_client_params["owner_username"], + latest_commit_sha=codecov_client_params["head_sha"], + ) + + # Pass this into format_plan_step_msg if they exist. Then combine the prompts + existing_test_design_response = self._get_test_design_summary( agent=agent, prompt=CodingUnitTestPrompts.format_find_unit_test_pattern_step_msg( @@ -58,7 +75,12 @@ def invoke(self, request: CodeUnitTestRequest) -> CodeUnitTestOutput | None: ) self._get_plan( - agent=agent, prompt=CodingUnitTestPrompts.format_plan_step_msg(diff_str=request.diff) + agent=agent, + prompt=CodingUnitTestPrompts.format_plan_step_msg( + diff_str=request.diff, + has_coverage_info=code_coverage_data, + has_test_result_info=test_result_data, + ), ) final_response = self._generate_tests( @@ -70,7 +92,6 @@ def invoke(self, request: CodeUnitTestRequest) -> CodeUnitTestOutput | None: if not final_response: return None - plan_steps_content = extract_text_inside_tags(final_response, "plan_steps") if len(plan_steps_content) == 0: diff --git a/src/seer/automation/codegen/unittest_step.py b/src/seer/automation/codegen/unittest_step.py index 3e4536be..7a4d4626 100644 --- a/src/seer/automation/codegen/unittest_step.py +++ b/src/seer/automation/codegen/unittest_step.py @@ -53,13 +53,22 @@ def _invoke(self, **kwargs): repo_client = self.context.get_repo_client() pr = repo_client.repo.get_pull(self.request.pr_id) - diff_content = repo_client.get_pr_diff_content(pr.url) + latest_commit_sha = repo_client.get_pr_head_sha(pr.url) + + codecov_client_params = { + "repo_name": self.request.repo_definition.name, + "pullid": self.request.pr_id, + "owner_username": self.request.repo_definition.owner, + "head_sha": latest_commit_sha, + } + unittest_output = UnitTestCodingComponent(self.context).invoke( CodeUnitTestRequest( diff=diff_content, - ) + ), + codecov_client_params=codecov_client_params, ) if unittest_output: