Skip to content

Commit

Permalink
set seed before each test (#5468)
Browse files Browse the repository at this point in the history
**Context:**

We've been seeing a lot of stochastic failures lately. We have random
tests that rely on the global numpy rng. Adding new tests or changing
around the order tests are run in can cause tests to randomly fail,
eating large amounts of CI and developer time.

**Description of the Change:**

Sets a global numpy seed before each test.

**Benefits:**

No more CI flakiness.

**Possible Drawbacks:**

Might still be a "special" seed, with failures on other seed values. We
should periodically update this seed just for testing robustness.

**Related GitHub Issues:**
  • Loading branch information
albi3ro committed May 1, 2024
1 parent 8df07c1 commit a7cd619
Show file tree
Hide file tree
Showing 5 changed files with 13 additions and 9 deletions.
6 changes: 6 additions & 0 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,12 @@ class DummyDevice(DefaultGaussian):
_operation_map["Kerr"] = lambda *x, **y: np.identity(2)


@pytest.fixture(autouse=True)
def set_numpy_seed():
np.random.seed(9872653)
yield


@pytest.fixture(scope="session")
def tol():
"""Numerical tolerance for equality tests."""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -718,7 +718,7 @@ def test_multiple_expectation_values(self, approx_order, strategy, validate):

assert isinstance(res[0], tuple)
assert len(res[0]) == 2
assert np.allclose(res[0], [-np.sin(x), 0], atol=0.1, rtol=0)
assert np.allclose(res[0], [-np.sin(x), 0], atol=0.15, rtol=0)
assert isinstance(res[0][0], numpy.ndarray)
assert isinstance(res[0][1], numpy.ndarray)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1305,7 +1305,7 @@ def test_non_involutory_variance_single_param(self):
for gradF in all_gradF:
assert isinstance(gradF, np.ndarray)
assert gradF.shape == ()
assert gradF == pytest.approx(expected, abs=1)
assert qml.math.allclose(gradF, expected, atol=2 * _herm_shot_vec_tol)

@flaky(max_runs=5)
def test_non_involutory_variance_multi_param(self):
Expand Down Expand Up @@ -1359,7 +1359,7 @@ def test_non_involutory_variance_multi_param(self):
# Note: the tolerances here are significantly higher than in usual tests
# due to the stochasticity of the test case
assert gradF[0] == pytest.approx(expected, abs=2)
assert gradF[1] == pytest.approx(expected, abs=1)
assert qml.math.allclose(gradF[1], expected, atol=1.5)

@flaky(max_runs=8)
def test_involutory_and_noninvolutory_variance_single_param(self):
Expand Down Expand Up @@ -1486,8 +1486,8 @@ def test_involutory_and_noninvolutory_variance_multi_param(self):

assert np.allclose(shot_vec_result[0][0], expected[0], atol=1)
assert np.allclose(shot_vec_result[0][1], expected[1], atol=1)
assert np.allclose(shot_vec_result[1][0], expected[2], atol=1)
assert np.allclose(shot_vec_result[1][1], expected[3], atol=1)
assert np.allclose(shot_vec_result[1][0], expected[2], atol=1.5)
assert np.allclose(shot_vec_result[1][1], expected[3], atol=1.5)

@pytest.mark.parametrize("ind", [0, 1])
def test_var_and_probs_single_param(self, ind):
Expand Down
4 changes: 1 addition & 3 deletions tests/transforms/test_hamiltonian_expand.py
Original file line number Diff line number Diff line change
Expand Up @@ -345,9 +345,7 @@ def test_processing_function_shot_vectors_broadcasting(self, H, expected, groupi
"""Tests that the processing function works with shot vectors, parameter broadcasting,
and grouping with different number of coefficients in each group"""

np.random.seed(824)
dev_with_shot_vector = qml.device("default.qubit", shots=[(8000, 4)])

dev_with_shot_vector = qml.device("default.qubit", shots=[(10000, 4)])
if grouping:
H.compute_grouping()

Expand Down
2 changes: 1 addition & 1 deletion tests/transforms/test_qcut.py
Original file line number Diff line number Diff line change
Expand Up @@ -2506,7 +2506,7 @@ def target_circuit(v):
qml.RX(2.3, wires=2)
return qml.expval(qml.PauliZ(wires=0) @ qml.PauliZ(wires=2))

dev = dev_fn(wires=2, shots=10000)
dev = dev_fn(wires=2, shots=20000)

@partial(qml.cut_circuit_mc, classical_processing_fn=fn)
@qml.qnode(dev)
Expand Down

0 comments on commit a7cd619

Please sign in to comment.