diff --git a/.github/workflows/interface-unit-tests.yml b/.github/workflows/interface-unit-tests.yml
index ad987251418..2e851d7f7ac 100644
--- a/.github/workflows/interface-unit-tests.yml
+++ b/.github/workflows/interface-unit-tests.yml
@@ -76,6 +76,14 @@ on:
required: false
type: boolean
default: false
+ python_warning_level:
+ description: Sets the default Python warning level as defined by https://docs.python.org/3/using/cmdline.html#envvar-PYTHONWARNINGS
+ required: false
+ type: string
+ default: 'default'
+env:
+ ACTIONS_RUNNER_DEBUG: true
+ ACTIONS_STEP_DEBUG: true
jobs:
determine_runner:
@@ -86,6 +94,20 @@ jobs:
default_runner: ubuntu-latest
force_large_runner: ${{ inputs.use_large_runner }}
+ disable-fail-fast-wae:
+ needs:
+ - determine_runner
+
+ runs-on: ${{ needs.determine_runner.outputs.runner_group }}
+ steps:
+ - name: Set fail-fast for WAE
+ id: mat_fail_fast
+ run: |
+ echo "fail_fast=${{ contains(inputs.python_warning_level, 'default') && 'default' || 'error' }}" >> $GITHUB_OUTPUT
+
+ outputs:
+ fail_fast: ${{ steps.mat_fail_fast.outputs.fail_fast }}
+
setup-ci-load:
needs:
- determine_runner
@@ -106,6 +128,13 @@ jobs:
"default": ["3.10"]
}
EOF
+ elif [ "${{ inputs.python_warning_level }}" == "error" ];
+ then
+ cat >python_versions.json <<-EOF
+ {
+ "default": ["3.10"]
+ }
+ EOF
else
cat >python_versions.json <<-EOF
{
@@ -191,7 +220,9 @@ jobs:
- setup-ci-load
- determine_runner
- default-dependency-versions
+ - disable-fail-fast-wae
strategy:
+ fail-fast: ${{ needs.disable-fail-fast-wae.outputs.fail_fast == 'default' }}
max-parallel: >-
${{
fromJSON(needs.setup-ci-load.outputs.matrix-max-parallel).torch-tests
@@ -218,6 +249,7 @@ jobs:
pytest_additional_args: --disable-opmath=${{ inputs.disable_new_opmath }}
pytest_coverage_flags: ${{ inputs.pytest_coverage_flags }}
pytest_markers: torch and not qcut and not finite-diff and not param-shift
+ pytest_additional_args: -W ${{ inputs.python_warning_level }} ${{ inputs.python_warning_level != 'default' && '--continue-on-collection-errors' || '' }}
requirements_file: ${{ github.event_name == 'schedule' && strategy.job-index == 0 && 'torch.txt' || '' }}
@@ -226,7 +258,9 @@ jobs:
- setup-ci-load
- determine_runner
- default-dependency-versions
+ - disable-fail-fast-wae
strategy:
+ fail-fast: ${{ contains(fromJSON(needs.disable-fail-fast-wae.outputs.fail_fast), 'default') }}
max-parallel: >-
${{
fromJSON(needs.setup-ci-load.outputs.matrix-max-parallel).autograd-tests
@@ -249,7 +283,7 @@ jobs:
additional_pip_packages: ${{ inputs.additional_python_packages }}
additional_pip_packages_post: ${{ needs.default-dependency-versions.outputs.pennylane-lightning-latest }}
pytest_coverage_flags: ${{ inputs.pytest_coverage_flags }}
- pytest_additional_args: --disable-opmath=${{ inputs.disable_new_opmath }}
+ pytest_additional_args: --disable-opmath=${{ inputs.disable_new_opmath }} -W ${{ inputs.python_warning_level }} ${{ inputs.python_warning_level != 'default' && '--continue-on-collection-errors' || '' }}
pytest_markers: autograd and not qcut and not finite-diff and not param-shift
@@ -258,7 +292,9 @@ jobs:
- setup-ci-load
- determine_runner
- default-dependency-versions
+ - disable-fail-fast-wae
strategy:
+ fail-fast: ${{ contains(fromJSON(needs.disable-fail-fast-wae.outputs.fail_fast), 'default') }}
max-parallel: >-
${{
fromJSON(needs.setup-ci-load.outputs.matrix-max-parallel).tf-tests
@@ -285,17 +321,18 @@ jobs:
additional_pip_packages_post: ${{ needs.default-dependency-versions.outputs.pennylane-lightning-latest }}
pytest_coverage_flags: ${{ inputs.pytest_coverage_flags }}
pytest_markers: tf and not qcut and not finite-diff and not param-shift
- pytest_additional_args: --splits 3 --group ${{ matrix.group }} --disable-opmath=${{ inputs.disable_new_opmath }}
+ pytest_additional_args: --splits 3 --group ${{ matrix.group }} --disable-opmath=${{ inputs.disable_new_opmath }} -W ${{ inputs.python_warning_level }} ${{ inputs.python_warning_level != 'default' && '--continue-on-collection-errors' || '' }}
pytest_durations_file_path: '.github/durations/tf_tests_durations.json'
requirements_file: ${{ github.event_name == 'schedule' && strategy.job-index == 0 && 'tf.txt' || '' }}
-
jax-tests:
needs:
- setup-ci-load
- determine_runner
- default-dependency-versions
+ - disable-fail-fast-wae
strategy:
+ fail-fast: ${{ contains(fromJSON(needs.disable-fail-fast-wae.outputs.fail_fast), 'default') }}
max-parallel: >-
${{
fromJSON(needs.setup-ci-load.outputs.matrix-max-parallel).jax-tests
@@ -322,7 +359,7 @@ jobs:
additional_pip_packages_post: ${{ needs.default-dependency-versions.outputs.pennylane-lightning-latest }}
pytest_coverage_flags: ${{ inputs.pytest_coverage_flags }}
pytest_markers: jax and not qcut and not finite-diff and not param-shift
- pytest_additional_args: --dist=loadscope --splits 4 --group ${{ matrix.group }} --disable-opmath=${{ inputs.disable_new_opmath }}
+ pytest_additional_args: --dist=loadscope --splits 4 --group ${{ matrix.group }} --disable-opmath=${{ inputs.disable_new_opmath }} -W ${{ inputs.python_warning_level }} ${{ inputs.python_warning_level != 'default' && '--continue-on-collection-errors' || '' }}
pytest_durations_file_path: '.github/durations/jax_tests_durations.json'
requirements_file: ${{ github.event_name == 'schedule' && strategy.job-index == 0 && 'jax.txt' || '' }}
@@ -332,7 +369,9 @@ jobs:
- setup-ci-load
- determine_runner
- default-dependency-versions
+ - disable-fail-fast-wae
strategy:
+ fail-fast: ${{ contains(fromJSON(needs.disable-fail-fast-wae.outputs.fail_fast), 'default') }}
max-parallel: >-
${{
fromJSON(needs.setup-ci-load.outputs.matrix-max-parallel).core-tests
@@ -357,7 +396,7 @@ jobs:
additional_pip_packages_post: ${{ needs.default-dependency-versions.outputs.pennylane-lightning-latest }}
pytest_coverage_flags: ${{ inputs.pytest_coverage_flags }}
pytest_markers: core and not qcut and not finite-diff and not param-shift
- pytest_additional_args: --splits 6 --group ${{ matrix.group }} --disable-opmath=${{ inputs.disable_new_opmath }}
+ pytest_additional_args: --splits 6 --group ${{ matrix.group }} --disable-opmath=${{ inputs.disable_new_opmath }} -W ${{ inputs.python_warning_level }} ${{ inputs.python_warning_level != 'default' && '--continue-on-collection-errors' || '' }}
pytest_durations_file_path: '.github/durations/core_tests_durations.json'
requirements_file: ${{ github.event_name == 'schedule' && strategy.job-index == 0 && 'core.txt' || '' }}
@@ -367,7 +406,9 @@ jobs:
- setup-ci-load
- determine_runner
- default-dependency-versions
+ - disable-fail-fast-wae
strategy:
+ fail-fast: ${{ contains(fromJSON(needs.disable-fail-fast-wae.outputs.fail_fast), 'default') }}
max-parallel: >-
${{
fromJSON(needs.setup-ci-load.outputs.matrix-max-parallel).all-interfaces-tests
@@ -395,7 +436,7 @@ jobs:
additional_pip_packages_post: ${{ needs.default-dependency-versions.outputs.pennylane-lightning-latest }}
pytest_coverage_flags: ${{ inputs.pytest_coverage_flags }}
pytest_markers: all_interfaces
- pytest_additional_args: --disable-opmath=${{ inputs.disable_new_opmath }}
+ pytest_additional_args: --disable-opmath=${{ inputs.disable_new_opmath }} -W ${{ inputs.python_warning_level }} ${{ inputs.python_warning_level != 'default' && '--continue-on-collection-errors' || '' }}
requirements_file: ${{ github.event_name == 'schedule' && strategy.job-index == 0 && 'all_interfaces.txt' || '' }}
@@ -404,7 +445,9 @@ jobs:
- setup-ci-load
- determine_runner
- default-dependency-versions
+ - disable-fail-fast-wae
strategy:
+ fail-fast: ${{ contains(fromJSON(needs.disable-fail-fast-wae.outputs.fail_fast), 'default') }}
max-parallel: >-
${{
fromJSON(needs.setup-ci-load.outputs.matrix-max-parallel).external-libraries-tests
@@ -426,7 +469,7 @@ jobs:
python_version: ${{ matrix.python-version }}
pytest_coverage_flags: ${{ inputs.pytest_coverage_flags }}
pytest_markers: external
- pytest_additional_args: --disable-opmath=${{ inputs.disable_new_opmath }}
+ pytest_additional_args: --disable-opmath=${{ inputs.disable_new_opmath }} -W ${{ inputs.python_warning_level }} ${{ inputs.python_warning_level != 'default' && '--continue-on-collection-errors' || '' }}
additional_pip_packages: |
pyzx matplotlib stim quimb mitiq pennylane-qiskit ply
${{ needs.default-dependency-versions.outputs.jax-version }}
@@ -448,7 +491,9 @@ jobs:
- setup-ci-load
- determine_runner
- default-dependency-versions
+ - disable-fail-fast-wae
strategy:
+ fail-fast: ${{ contains(fromJSON(needs.disable-fail-fast-wae.outputs.fail_fast), 'default') }}
max-parallel: >-
${{
fromJSON(needs.setup-ci-load.outputs.matrix-max-parallel).qcut-tests
@@ -470,7 +515,7 @@ jobs:
python_version: ${{ matrix.python-version }}
pytest_coverage_flags: ${{ inputs.pytest_coverage_flags }}
pytest_markers: qcut
- pytest_additional_args: --disable-opmath=${{ inputs.disable_new_opmath }}
+ pytest_additional_args: --disable-opmath=${{ inputs.disable_new_opmath }} -W ${{ inputs.python_warning_level }} ${{ inputs.python_warning_level != 'default' && '--continue-on-collection-errors' || '' }}
additional_pip_packages: |
kahypar==1.1.7
opt_einsum
@@ -486,7 +531,9 @@ jobs:
- setup-ci-load
- determine_runner
- default-dependency-versions
+ - disable-fail-fast-wae
strategy:
+ fail-fast: ${{ contains(fromJSON(needs.disable-fail-fast-wae.outputs.fail_fast), 'default') }}
max-parallel: >-
${{
fromJSON(needs.setup-ci-load.outputs.matrix-max-parallel).qchem-tests
@@ -510,7 +557,7 @@ jobs:
additional_pip_packages_post: ${{ needs.default-dependency-versions.outputs.pennylane-lightning-latest }}
pytest_coverage_flags: ${{ inputs.pytest_coverage_flags }}
pytest_markers: qchem
- pytest_additional_args: --disable-opmath=${{ inputs.disable_new_opmath }}
+ pytest_additional_args: --disable-opmath=${{ inputs.disable_new_opmath }} -W ${{ inputs.python_warning_level }} ${{ inputs.python_warning_level != 'default' && '--continue-on-collection-errors' || '' }}
additional_pip_packages: |
openfermionpyscf basis-set-exchange
${{ inputs.additional_python_packages }}
@@ -520,7 +567,9 @@ jobs:
- setup-ci-load
- determine_runner
- default-dependency-versions
+ - disable-fail-fast-wae
strategy:
+ fail-fast: ${{ contains(fromJSON(needs.disable-fail-fast-wae.outputs.fail_fast), 'default') }}
max-parallel: >-
${{
fromJSON(needs.setup-ci-load.outputs.matrix-max-parallel).gradients-tests
@@ -543,7 +592,7 @@ jobs:
branch: ${{ inputs.branch }}
coverage_artifact_name: gradients-${{ matrix.config.suite }}-coverage
python_version: ${{ matrix.python-version }}
- pytest_additional_args: --disable-opmath=${{ inputs.disable_new_opmath }}
+ pytest_additional_args: --disable-opmath=${{ inputs.disable_new_opmath }} -W ${{ inputs.python_warning_level }} ${{ inputs.python_warning_level != 'default' && '--continue-on-collection-errors' || '' }}
additional_pip_packages: |
${{ needs.default-dependency-versions.outputs.jax-version }}
${{ needs.default-dependency-versions.outputs.tensorflow-version }}
@@ -559,7 +608,9 @@ jobs:
- setup-ci-load
- determine_runner
- default-dependency-versions
+ - disable-fail-fast-wae
strategy:
+ fail-fast: ${{ contains(fromJSON(needs.disable-fail-fast-wae.outputs.fail_fast), 'default') }}
max-parallel: >-
${{
fromJSON(needs.setup-ci-load.outputs.matrix-max-parallel).data-tests
@@ -583,6 +634,7 @@ jobs:
pytest_coverage_flags: ${{ inputs.pytest_coverage_flags }}
pytest_additional_args: --disable-opmath=${{ inputs.disable_new_opmath }}
pytest_markers: data
+ pytest_additional_args: -W ${{ inputs.python_warning_level }} ${{ inputs.python_warning_level != 'default' && '--continue-on-collection-errors' || '' }}
additional_pip_packages: |
h5py
${{ inputs.additional_python_packages }}
@@ -593,7 +645,9 @@ jobs:
- setup-ci-load
- determine_runner
- default-dependency-versions
+ - disable-fail-fast-wae
strategy:
+ fail-fast: ${{ contains(fromJSON(needs.disable-fail-fast-wae.outputs.fail_fast), 'default') }}
max-parallel: >-
${{
fromJSON(needs.setup-ci-load.outputs.matrix-max-parallel).device-tests
@@ -628,7 +682,7 @@ jobs:
additional_pip_packages_post: ${{ needs.default-dependency-versions.outputs.pennylane-lightning-latest }}
pytest_test_directory: pennylane/devices/tests
pytest_coverage_flags: ${{ inputs.pytest_coverage_flags }}
- pytest_additional_args: --device=${{ matrix.config.device }} --shots=${{ matrix.config.shots }} --disable-opmath=${{ inputs.disable_new_opmath }}
+ pytest_additional_args: --device=${{ matrix.config.device }} --shots=${{ matrix.config.shots }} --disable-opmath=${{ inputs.disable_new_opmath }} -W ${{ inputs.python_warning_level }} ${{ inputs.python_warning_level != 'default' && '--continue-on-collection-errors' || '' }}
upload-to-codecov:
diff --git a/.github/workflows/package_warnings_as_errors.yml b/.github/workflows/package_warnings_as_errors.yml
new file mode 100644
index 00000000000..8bd97aed2e3
--- /dev/null
+++ b/.github/workflows/package_warnings_as_errors.yml
@@ -0,0 +1,31 @@
+name: Test-suite with Python warnings as errors
+on:
+ # Scheduled trigger every Sunday at 2:35am UTC
+ schedule:
+ - cron: '35 2 * * 0'
+ workflow_dispatch:
+
+concurrency:
+ group: warnings-as-errors-tests-${{ github.ref }}
+ cancel-in-progress: false
+
+jobs:
+ test-warnings-as-errors:
+ strategy:
+ fail-fast: false
+ max-parallel: 1
+ secrets:
+ codecov_token: ${{ secrets.CODECOV_TOKEN }}
+ uses: ./.github/workflows/interface-unit-tests.yml
+ with:
+ branch: ${{ github.ref }}
+ python_warning_level: 'error'
+ run_lightened_ci: false
+ skip_ci_test_jobs: |
+ tf-tests
+ all-interfaces-tests
+ external-libraries-tests
+ qcut-tests
+ qchem-tests
+ gradients-tests
+ device-tests
diff --git a/doc/releases/changelog-dev.md b/doc/releases/changelog-dev.md
index 1bddb979d80..6fc629d6e0e 100644
--- a/doc/releases/changelog-dev.md
+++ b/doc/releases/changelog-dev.md
@@ -42,6 +42,8 @@
Deprecations 👋
Documentation 📝
+* Add reporting of test warnings as failures.
+ [(#6217)](https://github.com/PennyLaneAI/pennylane/pull/6217)
Bug fixes 🐛
diff --git a/pennylane/devices/tests/pytest.ini b/pennylane/devices/tests/pytest.ini
index ba9755e1ca8..21b3865680d 100644
--- a/pennylane/devices/tests/pytest.ini
+++ b/pennylane/devices/tests/pytest.ini
@@ -1,4 +1,3 @@
[pytest]
markers =
skip_unsupported: skip a test if it uses an operation unsupported on a device
-addopts = --benchmark-disable
\ No newline at end of file
diff --git a/requirements-ci.txt b/requirements-ci.txt
index 4d7c028ba22..8a3808b7cdd 100644
--- a/requirements-ci.txt
+++ b/requirements-ci.txt
@@ -13,3 +13,4 @@ matplotlib
requests
rich
tomli # Drop once minimum Python version is 3.11
+pandas