From 30ff86bdef40dcb1699b38c5fec7e95f2145a548 Mon Sep 17 00:00:00 2001
From: Kevin Fairise <132568982+KevinFairise2@users.noreply.github.com>
Date: Wed, 5 Jun 2024 11:15:00 +0200
Subject: [PATCH] Fail if the Go test command has not been executed properly,
 even with TestWasher (#26181)

---
 tasks/testwasher.py                  | 20 ++++++++++++++++----
 tasks/unit-tests/testwasher_tests.py | 20 ++++++++++++++++----
 2 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/tasks/testwasher.py b/tasks/testwasher.py
index 3dd51c547f8e7..3a689b4943a22 100644
--- a/tasks/testwasher.py
+++ b/tasks/testwasher.py
@@ -22,13 +22,11 @@ def __init__(
 
         self.parse_flaky_file()
 
-    def get_non_flaky_failing_tests(self, module_path):
+    def get_non_flaky_failing_tests(self, failing_tests: dict, flaky_marked_tests: dict):
         """
         Parse the test output json file and compute the failing tests and the one known flaky
         """
 
-        failing_tests, flaky_marked_tests = self.parse_test_results(module_path)
-
         all_known_flakes = self.merge_known_flakes(flaky_marked_tests)
         non_flaky_failing_tests = defaultdict(set)
 
@@ -92,14 +90,28 @@ def process_module_results(self, module_results: list[ModuleTestResult]):
 
         should_succeed = True
         failed_tests = []
+        failed_command_modules = []
         for module_result in module_results:
-            if non_flaky_failing_tests := self.get_non_flaky_failing_tests(module_result.path):
+            failing_tests, flaky_marked_tests = self.parse_test_results(module_result.path)
+            non_flaky_failing_tests = self.get_non_flaky_failing_tests(
+                failing_tests=failing_tests, flaky_marked_tests=flaky_marked_tests
+            )
+            if (
+                not failing_tests and module_result.failed
+            ):  # In this case the Go test command failed on one of the modules but no test failed, it means that the test command itself failed (build errors,...)
+                should_succeed = False
+                failed_command_modules.append(module_result.path)
+            if non_flaky_failing_tests:
                 should_succeed = False
                 for package, tests in non_flaky_failing_tests.items():
                     failed_tests.extend(f"- {package} {test}" for test in tests)
         if failed_tests:
             print("The test command failed, the following tests failed and are not supposed to be flaky:")
             print("\n".join(sorted(failed_tests)))
+        if failed_command_modules:
+            print("The test command failed, before test execution on the following modules:")
+            print("\n".join(sorted(failed_command_modules)))
+            print("Please check the job logs for more information")
 
         return should_succeed
 
diff --git a/tasks/unit-tests/testwasher_tests.py b/tasks/unit-tests/testwasher_tests.py
index bcbfe56b199da..07805683898bc 100644
--- a/tasks/unit-tests/testwasher_tests.py
+++ b/tasks/unit-tests/testwasher_tests.py
@@ -10,7 +10,10 @@ def test_flaky_marked_failing_test(self):
             flakes_file_path="tasks/unit-tests/testdata/flakes_2.yaml",
         )
         module_path = "tasks/unit-tests/testdata"
-        non_flaky_failing_tests = test_washer_1.get_non_flaky_failing_tests(module_path)
+        failing_tests, marked_flaky_tests = test_washer_1.parse_test_results(module_path)
+        non_flaky_failing_tests = test_washer_1.get_non_flaky_failing_tests(
+            failing_tests=failing_tests, flaky_marked_tests=marked_flaky_tests
+        )
         self.assertEqual(non_flaky_failing_tests, {})
 
     def test_flakes_file_failing_test(self):
@@ -19,7 +22,10 @@ def test_flakes_file_failing_test(self):
             flakes_file_path="tasks/unit-tests/testdata/flakes_1.yaml",
         )
         module_path = "tasks/unit-tests/testdata"
-        non_flaky_failing_tests = test_washer_2.get_non_flaky_failing_tests(module_path)
+        failing_tests, marked_flaky_tests = test_washer_2.parse_test_results(module_path)
+        non_flaky_failing_tests = test_washer_2.get_non_flaky_failing_tests(
+            failing_tests=failing_tests, flaky_marked_tests=marked_flaky_tests
+        )
         self.assertEqual(non_flaky_failing_tests, {})
 
     def test_should_fail_failing_tests(self):
@@ -28,7 +34,10 @@ def test_should_fail_failing_tests(self):
             flakes_file_path="tasks/unit-tests/testdata/flakes_2.yaml",
         )
         module_path = "tasks/unit-tests/testdata"
-        non_flaky_failing_tests = test_washer_3.get_non_flaky_failing_tests(module_path)
+        failing_tests, marked_flaky_tests = test_washer_3.parse_test_results(module_path)
+        non_flaky_failing_tests = test_washer_3.get_non_flaky_failing_tests(
+            failing_tests=failing_tests, flaky_marked_tests=marked_flaky_tests
+        )
         self.assertEqual(non_flaky_failing_tests, {"github.com/DataDog/datadog-agent/pkg/gohai": {"TestGetPayload"}})
 
     def test_should_mark_parent_flaky(self):
@@ -37,7 +46,10 @@ def test_should_mark_parent_flaky(self):
             flakes_file_path="tasks/unit-tests/testdata/flakes_2.yaml",
         )
         module_path = "tasks/unit-tests/testdata"
-        non_flaky_failing_tests = test_washer.get_non_flaky_failing_tests(module_path)
+        failing_tests, marked_flaky_tests = test_washer.parse_test_results(module_path)
+        non_flaky_failing_tests = test_washer.get_non_flaky_failing_tests(
+            failing_tests=failing_tests, flaky_marked_tests=marked_flaky_tests
+        )
         self.assertEqual(
             non_flaky_failing_tests,
             {"github.com/DataDog/datadog-agent/test/new-e2e/tests/containers": {"TestEKSSuite/TestMemory"}},