CI: Report test results (#426)

* CI: Report test summary Signed-off-by: Brad Keryan <brad.keryan@ni.com> * CI: Consolidate test reporting Signed-off-by: Brad Keryan <brad.keryan@ni.com> * CI: Use bash to rename test results Signed-off-by: Brad Keryan <brad.keryan@ni.com> * CI: Revert filenames to kebab-case Both Tox envs and GitHub labels use kebab-case, so why fight it? Signed-off-by: Brad Keryan <brad.keryan@ni.com> * CI: Disable fail-fast for tests Signed-off-by: Brad Keryan <brad.keryan@ni.com> * CI: Fix renaming test results on Windows Signed-off-by: Brad Keryan <brad.keryan@ni.com> * CI: Use EnricoMi's test reporting action Signed-off-by: Brad Keryan <brad.keryan@ni.com> * CI: Don't forget to run report_test_results Signed-off-by: Brad Keryan <brad.keryan@ni.com> --------- Signed-off-by: Brad Keryan <brad.keryan@ni.com>
ni · Jun 30, 2023 · 41c22bb · 41c22bb
1 parent dd96c2c
commit 41c22bb
Show file tree

Hide file tree

Showing 6 changed files with 65 additions and 4 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -23,3 +23,8 @@ jobs:
     name: Run system tests
     uses: ./.github/workflows/run_system_tests.yml
     needs: [build, run_unit_tests]
+  report_test_results:
+    name: Report test results
+    uses: ./.github/workflows/report_test_results.yml
+    needs: [run_unit_tests, run_system_tests]
+    if: always()
diff --git a/.github/workflows/report_test_results.yml b/.github/workflows/report_test_results.yml
@@ -0,0 +1,28 @@
+name: Report test results
+
+on:
+  workflow_call:
+  workflow_dispatch:
+
+permissions:
+  checks: write
+  pull-requests: write
+
+jobs:
+  report_test_results:
+    name: Report test results
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v3
+      - name: Download test results
+        uses: actions/download-artifact@v3
+        with:
+          name: test_results
+          path: test_results
+      - name: List downloaded files
+        run: ls -lR
+      - name: Publish test results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        with:
+          files: "test_results/*.xml"
diff --git a/.github/workflows/run_system_tests.yml b/.github/workflows/run_system_tests.yml
@@ -14,11 +14,30 @@ jobs:
     strategy:
       matrix:
         configuration: ["win-10-py32", "win-10-py64"]
+      # Fail-fast skews the pass/fail ratio and seems to make pytest produce
+      # incomplete JUnit XML results.
+      fail-fast: false
     timeout-minutes: 90
     steps:
       - name: Check out repo
         uses: actions/checkout@v3
       - name: Install dependencies
         run: poetry install
       - name: Run system tests
-        run: poetry run tox
+        run: poetry run tox
+      - name: Rename test results
+        # TODO: Add Git Bash to the path on self-hosted Windows runners
+        run: Get-ChildItem test_results/*.xml | Rename-Item -NewName { $_.Name -replace '.xml','-${{ matrix.configuration }}.xml' }
+        if: always() && runner.os == 'Windows'
+      - name: Rename test results
+        run: |
+          for result in test_results/*.xml; do
+            mv $result ${result%.xml}-${{ matrix.configuration }}.xml
+          done
+        if: always() && runner.os != 'Windows'
+      - name: Upload test results
+        uses: actions/upload-artifact@v3
+        with:
+          name: test_results
+          path: test_results/*.xml
+        if: always()
diff --git a/.github/workflows/run_unit_tests.yml b/.github/workflows/run_unit_tests.yml
@@ -12,6 +12,9 @@ jobs:
       matrix:
         os: [ubuntu-latest, windows-latest]
         python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+      # Fail-fast skews the pass/fail ratio and seems to make pytest produce
+      # incomplete JUnit XML results.
+      fail-fast: false
     steps:
       - name: Check out repo
         uses: actions/checkout@v3
@@ -30,4 +33,10 @@ jobs:
           python -m pip install --upgrade pip
           poetry install
       - name: Run unit tests
-        run: poetry run pytest -v --cov=generated/nidaqmx tests/unit
+        run: poetry run pytest -v --cov=generated/nidaqmx --junitxml=test_results/unit-${{ matrix.os }}-py${{ matrix.python-version }}.xml tests/unit
+      - name: Upload test results
+        uses: actions/upload-artifact@v3
+        with:
+          name: test_results
+          path: test_results/*.xml
+        if: always()
diff --git a/.gitignore b/.gitignore
@@ -3,7 +3,7 @@ __pycache__/
 
 # Unit tests
 .tox/
-pytests-py*.xml
+test_results/
 
 # Coverage output
 .coverage

diff --git a/tox.ini b/tox.ini
@@ -18,7 +18,7 @@ setenv =
 commands =
    poetry run python --version
    poetry install -v {env:INSTALL_OPTS}
-   poetry run pytest --quiet --cov=generated/nidaqmx --cov-append --cov-report= --junitxml=pytests-{envname}.xml {env:PYTEST_OPTS} {posargs}
+   poetry run pytest --quiet --cov=generated/nidaqmx --cov-append --cov-report= --junitxml=test_results/system-{envname}.xml {env:PYTEST_OPTS} {posargs}
 
 [testenv:clean]
 commands = poetry run coverage erase