chore: add another test that our seeding system is good

by checking that the seed depends on the file name, function name and parameter
closes #zama-ai/concrete-ml-internal#4325

Makefile

@@ -210,9 +210,9 @@ spcc_internal: $(SPCC_DEPS)
 # -svv disables capturing of stdout/stderr and enables verbose output
 # --count N is to repeate all tests N times (with different seeds). Default is to COUNT=1.
 # --randomly-dont-reorganize is to prevent Pytest from shuffling the tests' order
-# --randomly-dont-reset-seed is to make sure that, if we run the same test several times (with
-# @pytest.mark.repeat(3)), different seeds are used, even if things are still deterministic using 
-# the main seed
+# --randomly-dont-reset-seed is important: if it was not there, the randomly package would reset
+# seeds to the same value, for all tests, resulting in same random's being taken in the tests, which
+# reduces a bit the impact / coverage of our tests
 # --capture=tee-sys is to make sure that, in case of crash, we can search for "Forcing seed to" in 
 # stdout in order to be able to reproduce the failed test using that seed
 # --cache-clear is to clear all Pytest's cache at before running the tests. This is done in order to 

script/make_utils/check_first_line_is_different_from_other_lines.py

@@ -0,0 +1,45 @@
+"""Helper to check the first line is different from all other lines."""
+
+import argparse
+from pathlib import Path
+
+
+def process_file(file_str: str):
+    """Helper to check the first line is different from all other lines.
+
+    Args:
+        file_str (str): the path to the file to process.
+
+    Returns:
+        True if everything went alright.
+
+    Raises:
+        ValueError: if the first line is equal to another line in the file
+
+    """
+    file_path = Path(file_str).resolve()
+
+    with open(file_path, "r", encoding="utf-8") as f:
+        file_content = f.readlines()
+
+    first_line = file_content[0]
+
+    for new_line in file_content[1:]:
+        is_equal_line = new_line == first_line
+        if is_equal_line:
+            raise ValueError("Error, the first line and another line are equal")
+
+    assert len(file_content) > 1
+    print(f"{file_path} looks good (number of lines: {len(file_content)})!")
+    return True
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        "Helper to check the first line is different from all other lines", allow_abbrev=False
+    )
+
+    parser.add_argument("--file", type=str, required=True, help="The log files to check")
+
+    cli_args = parser.parse_args()
+    process_file(cli_args.file)

script/make_utils/check_pytest_determinism.sh

@@ -70,4 +70,46 @@ echo ""
 echo "diff:"
 echo ""
 diff -u "${OUTPUT_DIRECTORY}/one.modified.txt" "${OUTPUT_DIRECTORY}/three.txt" --ignore-all-space --ignore-blank-lines --ignore-space-change
+
+# Run an execution and then check that the sub_seeds depends on the file name and the function name and parameters
+RANDOMLY_SEED=$RANDOMLY_SEED TEST=tests/seeding/test_seeding_system_file_a.py make pytest_one_single_cpu > "${OUTPUT_DIRECTORY}/seeds.txt"
+
+# This would not be readable
+# SC2181: Check exit code directly with e.g., 'if mycmd;', not indirectly with $?.
+# shellcheck disable=SC2181
+if [ $? -ne 0 ]
+then
+    echo "The commandline failed with:"
+    cat  "${OUTPUT_DIRECTORY}/seeds.txt"
+    exit 255
+fi
+
+RANDOMLY_SEED=$RANDOMLY_SEED TEST=tests/seeding/test_seeding_system_file_b.py make pytest_one_single_cpu >> "${OUTPUT_DIRECTORY}/seeds.txt"
+
+# This would not be readable
+# SC2181: Check exit code directly with e.g., 'if mycmd;', not indirectly with $?.
+# shellcheck disable=SC2181
+if [ $? -ne 0 ]
+then
+    echo "The commandline failed with:"
+    cat  "${OUTPUT_DIRECTORY}/seeds.txt"
+    exit 255
+fi
+
+# In the following test, we:
+#       log all Output's in seeds_outputs.txt, in a file of N lines: then, we'll check that
+#           the first Output is different from the 2nd to N-th once
+#       log all sub_seed's in seeds_sub_seed.txt, in a file of N lines: then, we'll check that
+#           the first sub_seed is different from the 2nd to N-th once
+#
+# The goal of these test is to check that, really, seeds and outputs are different, ie that the
+# filename or the function name or the parameters were really used in the seeding. Ideally, we
+# should check that all the outputs and seeds are different from each other, but it would be a long
+# test (ie, being in N**2 [or ideally N log(N)] instead of N), so it's not worth it
+grep "Output" "${OUTPUT_DIRECTORY}/seeds.txt" > "${OUTPUT_DIRECTORY}/seeds_outputs.txt"
+grep "sub_seed" "${OUTPUT_DIRECTORY}/seeds.txt" > "${OUTPUT_DIRECTORY}/seeds_sub_seed.txt"
+
+python script/make_utils/check_first_line_is_different_from_other_lines.py --file "${OUTPUT_DIRECTORY}/seeds_outputs.txt"
+python script/make_utils/check_first_line_is_different_from_other_lines.py --file "${OUTPUT_DIRECTORY}/seeds_sub_seed.txt"
+
 echo "Successful final check"

tests/seeding/test_seeding_system_file_a.py

@@ -0,0 +1,52 @@
+"""Tests for the torch to numpy module."""
+import random
+
+import numpy
+import pytest
+
+
+def body():
+    """Common function used in the tests, which picks random from numpy in different ways."""
+    numpy.set_printoptions(threshold=10000)
+    numpy.set_printoptions(linewidth=10000)
+
+    print("Output: ", end="")
+
+    # Python random
+    for _ in range(1):
+        print(random.randint(0, 1000), end="")
+
+    # Numpy random
+    for _ in range(1):
+        print(numpy.random.randint(0, 1000), end="")
+        print(numpy.random.uniform(-100, 100, size=(3, 3)).flatten(), end="")
+
+
+@pytest.mark.parametrize("parameters", [1, 2, 3])
+def test_bcm_seed_1(parameters):
+    """Test python and numpy seeding."""
+
+    assert parameters is not None
+    body()
+
+
+@pytest.mark.parametrize("parameters", [1, 3])
+def test_bcm_seed_2(parameters):
+    """Test python and numpy seeding."""
+
+    assert parameters is not None
+    body()
+
+
+@pytest.mark.parametrize("parameters", ["a", "b"])
+def test_bcm_seed_3(parameters):
+    """Test python and numpy seeding."""
+
+    assert parameters is not None
+    body()
+
+
+def test_bcm_seed_4():
+    """Test python and numpy seeding."""
+
+    body()

tests/seeding/test_seeding_system_file_b.py

@@ -0,0 +1,52 @@
+"""Tests for the torch to numpy module."""
+import random
+
+import numpy
+import pytest
+
+
+def body():
+    """Common function used in the tests, which picks random from numpy in different ways."""
+    numpy.set_printoptions(threshold=10000)
+    numpy.set_printoptions(linewidth=10000)
+
+    print("Output: ", end="")
+
+    # Python random
+    for _ in range(1):
+        print(random.randint(0, 1000), end="")
+
+    # Numpy random
+    for _ in range(1):
+        print(numpy.random.randint(0, 1000), end="")
+        print(numpy.random.uniform(-100, 100, size=(3, 3)).flatten(), end="")
+
+
+@pytest.mark.parametrize("parameters", [1, 2, 3])
+def test_bcm_seed_1(parameters):
+    """Test python and numpy seeding."""
+
+    assert parameters is not None
+    body()
+
+
+@pytest.mark.parametrize("parameters", [1, 3])
+def test_bcm_seed_2(parameters):
+    """Test python and numpy seeding."""
+
+    assert parameters is not None
+    body()
+
+
+@pytest.mark.parametrize("parameters", ["a", "b"])
+def test_bcm_seed_3(parameters):
+    """Test python and numpy seeding."""
+
+    assert parameters is not None
+    body()
+
+
+def test_bcm_seed_4():
+    """Test python and numpy seeding."""
+
+    body()