reftests: Skip non passing tests, enforce passing in the CI and impro…

…ving skip.
odoo · Jan 16, 2025 · 62ec727 · 62ec727
1 parent 23a0d47
commit 62ec727
Show file tree

Hide file tree

Showing 12 changed files with 3,366 additions and 3,686 deletions.
diff --git a/.github/workflows/checks-linux.yml b/.github/workflows/checks-linux.yml
@@ -1,6 +1,6 @@
 name: Checks (Ubuntu)
 
-on: [push, pull_request]
+on: [ push, pull_request ]
 
 jobs:
   ubuntu:
@@ -14,7 +14,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v3
         with:
-            python-version: '3.11'
+          python-version: '3.11'
 
       - name: Set up the build environment
         run: ./ck tools setup && ./ck tools doctor
@@ -28,5 +28,8 @@ jobs:
       - name: Test Userspace (Host)
         run: ./ck builder test
 
+      - name: Run the Reftests
+        run: ./ck reftests run --headless
+
       - name: Check for formatting errors
         run: ./meta/scripts/style-check.sh || echo "Please run ./meta/scripts/style-format.sh"
diff --git a/meta/plugins/reftest.py b/meta/plugins/reftest.py
@@ -61,6 +61,7 @@ def compareImages(
 
 class RefTestArgs(model.TargetArgs):
     glob: str = cli.arg("g", "glob")
+    headless: bool = cli.arg(None, "headless", "Run the tests without opening the report.")
     fast: str = cli.arg(
         None, "fast", "Proceed to the next test as soon as an error occurs."
     )
@@ -110,7 +111,9 @@ def update_temp_file(path, container, rendering):
     def getInfo(txt):
         return {prop: value for prop, value in REG_INFO.findall(txt)}
 
-    passed, failed = 0, 0
+    passed = 0
+    failed = 0
+    skipped = 0
 
     counter = 0
     for file in TESTS_DIR.glob(args.glob or "**/*.xhtml"):
@@ -123,15 +126,19 @@ def getInfo(txt):
 
         passCount = 0
         failCount = 0
+        skippedCount = 0
         for info, test in re.findall(r"""<test([^>]*)>([\w\W]+?)</test>""", content):
             props = getInfo(info)
             print(f"{vt100.WHITE}Test {props.get('name')!r}{vt100.RESET}")
             if "skip" in props:
+                skippedCount += 1
+                skipped += 1
+
                 report += f"""
                 <div>
                     <div id="case-{counter}" class="test skipped">
                         <h2>{props.get('name') or "Unamed"}</h2>
-                        <p>Skipped</p>
+                        <p>Test Skipped</p>
                     </div>
                 <div>
                 """
@@ -164,6 +171,9 @@ def getInfo(txt):
             ):
                 renderingProps = getInfo(info)
                 if "skip" in renderingProps:
+                    skippedCount += 1
+                    skipped += 1
+
                     print(f"{vt100.YELLOW}Skip test{vt100.RESET}")
                     continue
 
@@ -266,15 +276,15 @@ def getInfo(txt):
                         <h1>{props.get('name')}</h2>
                         <p>{props.get('help') or ""}</p>
                         <a href="{file}">Source</a>
-                        <span>{passCount} passed, {failCount} failed</span>
+                        <span>{passCount} passed, {failCount} failed and {skippedCount} skipped</span>
                     </div>
                     {test_report}
                 </div>
                 """
     report += f"""
         <footer>
         <p class="witty">{fetchMessage(args, 'witty' if failed else 'nice')}</p>
-        <p> Failed {failed} tests, Passed {passed} tests</p>
+        <p> Failed {failed} tests, Passed {passed} tests, Skipped {skipped}</p>
         </footer>
     """
 
@@ -454,18 +464,21 @@ def getInfo(txt):
     with (TEST_REPORT / "report.html").open("w") as f:
         f.write(report)
 
+    if not args.headless:
+        if shell.which("xdg-open"):
+            shell.exec("xdg-open", str(TEST_REPORT / "report.html"))
+        elif shell.which("open"):
+            shell.exec("open", str(TEST_REPORT / "report.html"))
+
     print()
     if failed:
         print(f"{vt100.BRIGHT_GREEN}// {fetchMessage(args, 'witty')}{vt100.RESET}")
         print(
             f"{vt100.RED}Failed {failed} tests{vt100.RESET}, {vt100.GREEN}Passed {passed} tests{vt100.RESET}"
         )
+        print(f"Report: {TEST_REPORT / 'report.html'}")
+        raise RuntimeError("Some tests failed")
     else:
         print(f"{vt100.GREEN}// {fetchMessage(args, 'nice')}{vt100.RESET}")
         print(f"{vt100.GREEN}All tests passed{vt100.RESET}")
-    print(f"Report: {TEST_REPORT / 'report.html'}")
-
-    if shell.which("xdg-open"):
-        shell.exec("xdg-open", str(TEST_REPORT / "report.html"))
-    elif shell.which("open"):
-        shell.exec("open", str(TEST_REPORT / "report.html"))
+        print(f"Report: {TEST_REPORT / 'report.html'}")