From 41c22bb67d3dd8a4dc64528cb53adfda0f2e100c Mon Sep 17 00:00:00 2001 From: Brad Keryan Date: Fri, 30 Jun 2023 11:39:28 -0500 Subject: [PATCH] CI: Report test results (#426) * CI: Report test summary Signed-off-by: Brad Keryan * CI: Consolidate test reporting Signed-off-by: Brad Keryan * CI: Use bash to rename test results Signed-off-by: Brad Keryan * CI: Revert filenames to kebab-case Both Tox envs and GitHub labels use kebab-case, so why fight it? Signed-off-by: Brad Keryan * CI: Disable fail-fast for tests Signed-off-by: Brad Keryan * CI: Fix renaming test results on Windows Signed-off-by: Brad Keryan * CI: Use EnricoMi's test reporting action Signed-off-by: Brad Keryan * CI: Don't forget to run report_test_results Signed-off-by: Brad Keryan --------- Signed-off-by: Brad Keryan --- .github/workflows/CI.yml | 5 ++++ .github/workflows/report_test_results.yml | 28 +++++++++++++++++++++++ .github/workflows/run_system_tests.yml | 21 ++++++++++++++++- .github/workflows/run_unit_tests.yml | 11 ++++++++- .gitignore | 2 +- tox.ini | 2 +- 6 files changed, 65 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/report_test_results.yml diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 5d0dd401..bee61bad 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -23,3 +23,8 @@ jobs: name: Run system tests uses: ./.github/workflows/run_system_tests.yml needs: [build, run_unit_tests] + report_test_results: + name: Report test results + uses: ./.github/workflows/report_test_results.yml + needs: [run_unit_tests, run_system_tests] + if: always() \ No newline at end of file diff --git a/.github/workflows/report_test_results.yml b/.github/workflows/report_test_results.yml new file mode 100644 index 00000000..bdf88e7f --- /dev/null +++ b/.github/workflows/report_test_results.yml @@ -0,0 +1,28 @@ +name: Report test results + +on: + workflow_call: + workflow_dispatch: + +permissions: + checks: write + pull-requests: write + +jobs: + report_test_results: + name: Report test results + runs-on: ubuntu-latest + steps: + - name: Check out repo + uses: actions/checkout@v3 + - name: Download test results + uses: actions/download-artifact@v3 + with: + name: test_results + path: test_results + - name: List downloaded files + run: ls -lR + - name: Publish test results + uses: EnricoMi/publish-unit-test-result-action@v2 + with: + files: "test_results/*.xml" diff --git a/.github/workflows/run_system_tests.yml b/.github/workflows/run_system_tests.yml index 224710bc..1f012174 100644 --- a/.github/workflows/run_system_tests.yml +++ b/.github/workflows/run_system_tests.yml @@ -14,6 +14,9 @@ jobs: strategy: matrix: configuration: ["win-10-py32", "win-10-py64"] + # Fail-fast skews the pass/fail ratio and seems to make pytest produce + # incomplete JUnit XML results. + fail-fast: false timeout-minutes: 90 steps: - name: Check out repo @@ -21,4 +24,20 @@ jobs: - name: Install dependencies run: poetry install - name: Run system tests - run: poetry run tox \ No newline at end of file + run: poetry run tox + - name: Rename test results + # TODO: Add Git Bash to the path on self-hosted Windows runners + run: Get-ChildItem test_results/*.xml | Rename-Item -NewName { $_.Name -replace '.xml','-${{ matrix.configuration }}.xml' } + if: always() && runner.os == 'Windows' + - name: Rename test results + run: | + for result in test_results/*.xml; do + mv $result ${result%.xml}-${{ matrix.configuration }}.xml + done + if: always() && runner.os != 'Windows' + - name: Upload test results + uses: actions/upload-artifact@v3 + with: + name: test_results + path: test_results/*.xml + if: always() diff --git a/.github/workflows/run_unit_tests.yml b/.github/workflows/run_unit_tests.yml index 97f0b413..3369e90a 100644 --- a/.github/workflows/run_unit_tests.yml +++ b/.github/workflows/run_unit_tests.yml @@ -12,6 +12,9 @@ jobs: matrix: os: [ubuntu-latest, windows-latest] python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + # Fail-fast skews the pass/fail ratio and seems to make pytest produce + # incomplete JUnit XML results. + fail-fast: false steps: - name: Check out repo uses: actions/checkout@v3 @@ -30,4 +33,10 @@ jobs: python -m pip install --upgrade pip poetry install - name: Run unit tests - run: poetry run pytest -v --cov=generated/nidaqmx tests/unit + run: poetry run pytest -v --cov=generated/nidaqmx --junitxml=test_results/unit-${{ matrix.os }}-py${{ matrix.python-version }}.xml tests/unit + - name: Upload test results + uses: actions/upload-artifact@v3 + with: + name: test_results + path: test_results/*.xml + if: always() \ No newline at end of file diff --git a/.gitignore b/.gitignore index 1ac33f20..c71a6d43 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ __pycache__/ # Unit tests .tox/ -pytests-py*.xml +test_results/ # Coverage output .coverage diff --git a/tox.ini b/tox.ini index 65567a5d..f9973772 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ setenv = commands = poetry run python --version poetry install -v {env:INSTALL_OPTS} - poetry run pytest --quiet --cov=generated/nidaqmx --cov-append --cov-report= --junitxml=pytests-{envname}.xml {env:PYTEST_OPTS} {posargs} + poetry run pytest --quiet --cov=generated/nidaqmx --cov-append --cov-report= --junitxml=test_results/system-{envname}.xml {env:PYTEST_OPTS} {posargs} [testenv:clean] commands = poetry run coverage erase