Enable warnings as errors for weekly run in test suite

.github/workflows/interface-unit-tests.yml

@@ -47,6 +47,11 @@ on:
  required: false
  type: boolean
  default: false
+ python_warning_level:
+ description: Sets the default Python warning level as defined by https://docs.python.org/3/using/cmdline.html#envvar-PYTHONWARNINGS
+ required: true
+ type: string
+ default: 'default'
 
 jobs:
  setup-ci-load:
@@ -168,6 +173,7 @@ jobs:
  pytest_markers: torch and not qcut and not finite-diff and not param-shift
  requirements_file: ${{ github.event_name == 'schedule' && strategy.job-index == 0 && 'torch.txt' || '' }}
  disable_new_opmath: ${{ inputs.disable_new_opmath }}
+ python_warning_level: ${{ inputs.python_warning_level }}
 
 
  autograd-tests:
@@ -200,6 +206,7 @@ jobs:
  pytest_coverage_flags: ${{ inputs.pytest_coverage_flags }}
  pytest_markers: autograd and not qcut and not finite-diff and not param-shift
  disable_new_opmath: ${{ inputs.disable_new_opmath }}
+ python_warning_level: ${{ inputs.python_warning_level }}
 
 
  tf-tests:
@@ -238,6 +245,7 @@ jobs:
  additional_pip_packages: pytest-split
  requirements_file: ${{ github.event_name == 'schedule' && strategy.job-index == 0 && 'tf.txt' || '' }}
  disable_new_opmath: ${{ inputs.disable_new_opmath }}
+ python_warning_level: ${{ inputs.python_warning_level }}
 
 
  jax-tests:
@@ -276,6 +284,7 @@ jobs:
  additional_pip_packages: pytest-split
  requirements_file: ${{ github.event_name == 'schedule' && strategy.job-index == 0 && 'jax.txt' || '' }}
  disable_new_opmath: ${{ inputs.disable_new_opmath }}
+ python_warning_level: ${{ inputs.python_warning_level }}
 
 
  core-tests:
@@ -314,6 +323,7 @@ jobs:
  additional_pip_packages: pytest-split
  requirements_file: ${{ github.event_name == 'schedule' && strategy.job-index == 0 && 'core.txt' || '' }}
  disable_new_opmath: ${{ inputs.disable_new_opmath }}
+ python_warning_level: ${{ inputs.python_warning_level }}
 
 
  all-interfaces-tests:
@@ -347,6 +357,7 @@ jobs:
  pytest_markers: all_interfaces
  requirements_file: ${{ github.event_name == 'schedule' && strategy.job-index == 0 && 'all_interfaces.txt' || '' }}
  disable_new_opmath: ${{ inputs.disable_new_opmath }}
+ python_warning_level: ${{ inputs.python_warning_level }}
 
 
  external-libraries-tests:
@@ -386,6 +397,7 @@ jobs:
  additional_pip_packages: pyzx matplotlib stim quimb mitiq pennylane-qiskit ply
  requirements_file: ${{ github.event_name == 'schedule' && strategy.job-index == 0 && 'external.txt' || '' }}
  disable_new_opmath: ${{ inputs.disable_new_opmath }}
+ python_warning_level: ${{ inputs.python_warning_level }}
 
 
  qcut-tests:
@@ -419,6 +431,7 @@ jobs:
  pytest_markers: qcut
  additional_pip_packages: kahypar==1.1.7 opt_einsum
  disable_new_opmath: ${{ inputs.disable_new_opmath }}
+ python_warning_level: ${{ inputs.python_warning_level }}
 
 
  qchem-tests:
@@ -452,6 +465,7 @@ jobs:
  pytest_markers: qchem
  additional_pip_packages: openfermionpyscf basis-set-exchange
  disable_new_opmath: ${{ inputs.disable_new_opmath }}
+ python_warning_level: ${{ inputs.python_warning_level }}
 
  gradients-tests:
  needs:
@@ -486,6 +500,7 @@ jobs:
  pytest_coverage_flags: ${{ inputs.pytest_coverage_flags }}
  pytest_markers: ${{ matrix.config.suite }}
  disable_new_opmath: ${{ inputs.disable_new_opmath }}
+ python_warning_level: ${{ inputs.python_warning_level }}
 
 
  data-tests:
@@ -519,6 +534,7 @@ jobs:
  pytest_markers: data
  additional_pip_packages: h5py
  disable_new_opmath: ${{ inputs.disable_new_opmath }}
+ python_warning_level: ${{ inputs.python_warning_level }}
 
 
  device-tests:
@@ -559,6 +575,7 @@ jobs:
  pytest_coverage_flags: ${{ inputs.pytest_coverage_flags }}
  pytest_additional_args: --device=${{ matrix.config.device }} --shots=${{ matrix.config.shots }}
  disable_new_opmath: ${{ inputs.disable_new_opmath }}
+ python_warning_level: ${{ inputs.python_warning_level }}
 
 
  upload-to-codecov:

.github/workflows/package_warnings_as_errors.yml

@@ -0,0 +1,25 @@
+name: Test-suite with warnings as error
+on:
+ # Scheduled trigger every Sunday at 2:35am UTC
+ schedule:
+ - cron: '35 2 * * 0'
+ workflow_dispatch:
+
+concurrency:
+ group: unit-tests-${{ github.ref }}
+ cancel-in-progress: true
+
+env:
+ DURATIONS_BRANCH: bot/durations-update
+
+jobs:
+ test-warnings-as-errors:
+ uses: ./.github/workflows/interface-unit-tests.yml
+ secrets:
+ codecov_token: ${{ secrets.CODECOV_TOKEN }}
+ with:
+ branch: ${{ github.ref }}
+ pipeline_mode: 'warnings-as-errors'
+ run_lightened_ci: false
+ skip_ci_test_jobs: 'qcut-tests'
+ python_warning_level: 'error'

.github/workflows/unit-test.yml

@@ -114,6 +114,11 @@ on:
  required: false
  type: string
  default: "False"
+ python_warning_level:
+ description: Sets the default Python warning level as defined by https://docs.python.org/3/using/cmdline.html#envvar-PYTHONWARNINGS
+ required: true
+ type: string
+ default: 'default'
 
 jobs:
  test:
@@ -187,6 +192,7 @@ jobs:
  - name: Run PennyLane Unit Tests
  if: steps.continue.outputs.confirm == 'true'
  env:
+ PYTHONWARNINGS: ${{ inputs.python_warning_level }} 
  PYTEST_MARKER: ${{ inputs.pytest_markers != '' && format('-m "{0}"', inputs.pytest_markers) || '' }}
  COV_CORE_SOURCE: pennylane
  COV_CORE_CONFIG: .coveragerc

doc/releases/changelog-dev.md

@@ -85,6 +85,9 @@
 
 <h3>Documentation 📝</h3>
 
+* Add reporting of test warnings as failures.
+ [(#6217)](https://github.com/PennyLaneAI/pennylane/pull/6217)
+
 <h3>Bug fixes 🐛</h3>
 
 * Fix a bug where zero-valued JVPs were calculated wrongly in the presence of shot vectors.

pennylane/devices/tests/pytest.ini

@@ -1,4 +1,4 @@
 [pytest]
 markers =
  skip_unsupported: skip a test if it uses an operation unsupported on a device
-addopts = --benchmark-disable
+addopts = --benchmark-disable

tests/pytest.ini

@@ -19,12 +19,5 @@ markers =
  logging: marks tests for pennylane logging
  external: marks tests that require external packages such as matplotlib and PyZX
  catalyst: marks tests for catalyst testing (select with '-m "catalyst"')
-filterwarnings =
- ignore::DeprecationWarning:autograd.numpy.numpy_wrapper
- ignore:Casting complex values to real::autograd.numpy.numpy_wrapper
- ignore:Casting complex values to real discards the imaginary part:UserWarning:torch.autograd
- ignore:Call to deprecated create function:DeprecationWarning
- ignore:the imp module is deprecated:DeprecationWarning
- error::pennylane.PennyLaneDeprecationWarning
 addopts = --benchmark-disable
 xfail_strict=true