From bab4326f76fd7aa35df0c74805b4fffce6f376bf Mon Sep 17 00:00:00 2001 From: Benoit Chevallier-Mames Date: Wed, 20 Mar 2024 16:28:43 +0100 Subject: [PATCH] chore: add another test that our seeding system is good by checking that the seed depends on the file name, function name and parameter closes #https://github.com/zama-ai/concrete-ml-internal/issues/4325 --- Makefile | 6 +-- ...irst_line_is_different_from_other_lines.py | 45 ++++++++++++++++ script/make_utils/check_pytest_determinism.sh | 42 +++++++++++++++ tests/seeding/test_seeding_system_file_a.py | 52 +++++++++++++++++++ tests/seeding/test_seeding_system_file_b.py | 52 +++++++++++++++++++ 5 files changed, 194 insertions(+), 3 deletions(-) create mode 100644 script/make_utils/check_first_line_is_different_from_other_lines.py create mode 100644 tests/seeding/test_seeding_system_file_a.py create mode 100644 tests/seeding/test_seeding_system_file_b.py diff --git a/Makefile b/Makefile index 2ec9aae83..ce01b9e67 100644 --- a/Makefile +++ b/Makefile @@ -210,9 +210,9 @@ spcc_internal: $(SPCC_DEPS) # -svv disables capturing of stdout/stderr and enables verbose output # --count N is to repeate all tests N times (with different seeds). Default is to COUNT=1. # --randomly-dont-reorganize is to prevent Pytest from shuffling the tests' order -# --randomly-dont-reset-seed is to make sure that, if we run the same test several times (with -# @pytest.mark.repeat(3)), different seeds are used, even if things are still deterministic using -# the main seed +# --randomly-dont-reset-seed is important: if it was not there, the randomly package would reset +# seeds to the same value, for all tests, resulting in same random's being taken in the tests, which +# reduces a bit the impact / coverage of our tests # --capture=tee-sys is to make sure that, in case of crash, we can search for "Forcing seed to" in # stdout in order to be able to reproduce the failed test using that seed # --cache-clear is to clear all Pytest's cache at before running the tests. This is done in order to diff --git a/script/make_utils/check_first_line_is_different_from_other_lines.py b/script/make_utils/check_first_line_is_different_from_other_lines.py new file mode 100644 index 000000000..65e90f489 --- /dev/null +++ b/script/make_utils/check_first_line_is_different_from_other_lines.py @@ -0,0 +1,45 @@ +"""Helper to check the first line is different from all other lines.""" + +import argparse +from pathlib import Path + + +def process_file(file_str: str): + """Helper to check the first line is different from all other lines. + + Args: + file_str (str): the path to the file to process. + + Returns: + True if everything went alright. + + Raises: + ValueError: if the first line is equal to another line in the file + + """ + file_path = Path(file_str).resolve() + + with open(file_path, "r", encoding="utf-8") as f: + file_content = f.readlines() + + first_line = file_content[0] + + for new_line in file_content[1:]: + is_equal_line = new_line == first_line + if is_equal_line: + raise ValueError("Error, the first line and another line are equal") + + assert len(file_content) > 1 + print(f"{file_path} looks good (number of lines: {len(file_content)})!") + return True + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + "Helper to check the first line is different from all other lines", allow_abbrev=False + ) + + parser.add_argument("--file", type=str, required=True, help="The log files to check") + + cli_args = parser.parse_args() + process_file(cli_args.file) diff --git a/script/make_utils/check_pytest_determinism.sh b/script/make_utils/check_pytest_determinism.sh index 0c244ef85..00adf5e1e 100755 --- a/script/make_utils/check_pytest_determinism.sh +++ b/script/make_utils/check_pytest_determinism.sh @@ -70,4 +70,46 @@ echo "" echo "diff:" echo "" diff -u "${OUTPUT_DIRECTORY}/one.modified.txt" "${OUTPUT_DIRECTORY}/three.txt" --ignore-all-space --ignore-blank-lines --ignore-space-change + +# Run an execution and then check that the sub_seeds depends on the file name and the function name and parameters +RANDOMLY_SEED=$RANDOMLY_SEED TEST=tests/seeding/test_seeding_system_file_a.py make pytest_one_single_cpu > "${OUTPUT_DIRECTORY}/seeds.txt" + +# This would not be readable +# SC2181: Check exit code directly with e.g., 'if mycmd;', not indirectly with $?. +# shellcheck disable=SC2181 +if [ $? -ne 0 ] +then + echo "The commandline failed with:" + cat "${OUTPUT_DIRECTORY}/seeds.txt" + exit 255 +fi + +RANDOMLY_SEED=$RANDOMLY_SEED TEST=tests/seeding/test_seeding_system_file_b.py make pytest_one_single_cpu >> "${OUTPUT_DIRECTORY}/seeds.txt" + +# This would not be readable +# SC2181: Check exit code directly with e.g., 'if mycmd;', not indirectly with $?. +# shellcheck disable=SC2181 +if [ $? -ne 0 ] +then + echo "The commandline failed with:" + cat "${OUTPUT_DIRECTORY}/seeds.txt" + exit 255 +fi + +# In the following test, we: +# log all Output's in seeds_outputs.txt, in a file of N lines: then, we'll check that +# the first Output is different from the 2nd to N-th once +# log all sub_seed's in seeds_sub_seed.txt, in a file of N lines: then, we'll check that +# the first sub_seed is different from the 2nd to N-th once +# +# The goal of these test is to check that, really, seeds and outputs are different, ie that the +# filename or the function name or the parameters were really used in the seeding. Ideally, we +# should check that all the outputs and seeds are different from each other, but it would be a long +# test (ie, being in N**2 [or ideally N log(N)] instead of N), so it's not worth it +grep "Output" "${OUTPUT_DIRECTORY}/seeds.txt" > "${OUTPUT_DIRECTORY}/seeds_outputs.txt" +grep "sub_seed" "${OUTPUT_DIRECTORY}/seeds.txt" > "${OUTPUT_DIRECTORY}/seeds_sub_seed.txt" + +python script/make_utils/check_first_line_is_different_from_other_lines.py --file "${OUTPUT_DIRECTORY}/seeds_outputs.txt" +python script/make_utils/check_first_line_is_different_from_other_lines.py --file "${OUTPUT_DIRECTORY}/seeds_sub_seed.txt" + echo "Successful final check" diff --git a/tests/seeding/test_seeding_system_file_a.py b/tests/seeding/test_seeding_system_file_a.py new file mode 100644 index 000000000..8bbac1bab --- /dev/null +++ b/tests/seeding/test_seeding_system_file_a.py @@ -0,0 +1,52 @@ +"""Tests for the torch to numpy module.""" +import random + +import numpy +import pytest + + +def body(): + """Common function used in the tests, which picks random from numpy in different ways.""" + numpy.set_printoptions(threshold=10000) + numpy.set_printoptions(linewidth=10000) + + print("Output: ", end="") + + # Python random + for _ in range(1): + print(random.randint(0, 1000), end="") + + # Numpy random + for _ in range(1): + print(numpy.random.randint(0, 1000), end="") + print(numpy.random.uniform(-100, 100, size=(3, 3)).flatten(), end="") + + +@pytest.mark.parametrize("parameters", [1, 2, 3]) +def test_bcm_seed_1(parameters): + """Test python and numpy seeding.""" + + assert parameters is not None + body() + + +@pytest.mark.parametrize("parameters", [1, 3]) +def test_bcm_seed_2(parameters): + """Test python and numpy seeding.""" + + assert parameters is not None + body() + + +@pytest.mark.parametrize("parameters", ["a", "b"]) +def test_bcm_seed_3(parameters): + """Test python and numpy seeding.""" + + assert parameters is not None + body() + + +def test_bcm_seed_4(): + """Test python and numpy seeding.""" + + body() diff --git a/tests/seeding/test_seeding_system_file_b.py b/tests/seeding/test_seeding_system_file_b.py new file mode 100644 index 000000000..8bbac1bab --- /dev/null +++ b/tests/seeding/test_seeding_system_file_b.py @@ -0,0 +1,52 @@ +"""Tests for the torch to numpy module.""" +import random + +import numpy +import pytest + + +def body(): + """Common function used in the tests, which picks random from numpy in different ways.""" + numpy.set_printoptions(threshold=10000) + numpy.set_printoptions(linewidth=10000) + + print("Output: ", end="") + + # Python random + for _ in range(1): + print(random.randint(0, 1000), end="") + + # Numpy random + for _ in range(1): + print(numpy.random.randint(0, 1000), end="") + print(numpy.random.uniform(-100, 100, size=(3, 3)).flatten(), end="") + + +@pytest.mark.parametrize("parameters", [1, 2, 3]) +def test_bcm_seed_1(parameters): + """Test python and numpy seeding.""" + + assert parameters is not None + body() + + +@pytest.mark.parametrize("parameters", [1, 3]) +def test_bcm_seed_2(parameters): + """Test python and numpy seeding.""" + + assert parameters is not None + body() + + +@pytest.mark.parametrize("parameters", ["a", "b"]) +def test_bcm_seed_3(parameters): + """Test python and numpy seeding.""" + + assert parameters is not None + body() + + +def test_bcm_seed_4(): + """Test python and numpy seeding.""" + + body()