Add seed checker bypass flag

mlcommons · Sep 19, 2024 · d2bce16 · d2bce16
1 parent ac0096c
commit d2bce16
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 12 deletions.
diff --git a/mlperf_logging/package_checker/package_checker.py b/mlperf_logging/package_checker/package_checker.py
@@ -34,7 +34,7 @@ def _print_divider_bar():
 
 
 def check_training_result_files(folder, usage, ruleset, quiet, werror,
-                                rcp_bypass, rcp_bert_train_samples):
+                                rcp_bypass, rcp_bert_train_samples, seed_checker_bypass):
     """Checks all result files for compliance.
 
     Args:
@@ -44,6 +44,7 @@ def check_training_result_files(folder, usage, ruleset, quiet, werror,
     allowed_benchmarks = get_allowed_benchmarks(usage, ruleset)
     benchmark_file_counts = get_result_file_counts(usage)
     global_rcp_bypass = rcp_bypass
+    global_seed_checker_bypass = seed_checker_bypass
 
     seed_checker = SeedChecker(ruleset)
     too_many_errors = False
@@ -58,12 +59,15 @@ def check_training_result_files(folder, usage, ruleset, quiet, werror,
         # Set system wide rcp-bypass
         params_path = os.path.join(system_folder, "package_checker_params")
         system_rcp_bypass = False
+        system_seed_checker_bypass = False
         if os.path.exists(params_path):
             with open(params_path) as f:
                 lines = f.readlines()
                 for line in lines:
                     if line == "rcp-bypass":
                         system_rcp_bypass = True
+                    if line == "seed-checker-bypass":
+                        system_seed_checker_bypass = True
         for benchmark_folder in benchmark_folders:
             folder_parts = benchmark_folder.split('/')
             benchmark = folder_parts[-1]
@@ -98,13 +102,16 @@ def check_training_result_files(folder, usage, ruleset, quiet, werror,
             params_files = []
             params_path = os.path.join(benchmark_folder, "package_checker_params")
             result_rcp_bypass = False
+            result_seed_checker_bypass = False
             if os.path.exists(params_path):
                 params_files.append(params_path)
                 with open(params_path) as f:
                     lines = f.readlines()
                     for line in lines:
                         if line == "rcp-bypass":
                             result_rcp_bypass = True
+                        if line == "seed-checker-bypass":
+                            result_seed_checker_bypass = True
 
             # Find all source codes for this benchmark.
             source_files = find_source_files_under(
@@ -176,7 +183,8 @@ def check_training_result_files(folder, usage, ruleset, quiet, werror,
 
             # Check if each run use unique seeds.
             if ruleset in {'1.0.0', '1.1.0', '2.0.0', '2.1.0', '3.0.0', '3.1.0', '4.0.0', '4.1.0'} and division == 'closed':
-                if not seed_checker.check_seeds(result_files):
+                seed_checker_bypass = (global_seed_checker_bypass or system_seed_checker_bypass or result_seed_checker_bypass)
+                if not seed_checker.check_seeds(result_files, seed_checker_bypass):
                     too_many_errors = True
                     logging.error('Seed checker failed')
 

diff --git a/mlperf_logging/package_checker/seed_checker.py b/mlperf_logging/package_checker/seed_checker.py
@@ -123,7 +123,7 @@ def _has_seed_keyword(self, source_file):
                     return True
         return False
 
-    def check_seeds(self, result_files):
+    def check_seeds(self, result_files, seed_checker_bypass = False):
         """ Check the seeds for a specific benchmark submission.
 
         Args:
@@ -133,13 +133,16 @@ def check_seeds(self, result_files):
         """
         _print_divider_bar()
         logging.info(" Running Seed Checker")
-        error_messages = self._assert_unique_seed_per_run(
-            result_files
-        )
-
-        if len(error_messages) > 0:
-            logging.error(" Seed checker failed and found the following errors: %s", '\n'.join(error_messages))
-            #print("Seed checker failed and found the following "
-            #      "errors:\n{}".format('\n'.join(error_messages)))
-            return False
+        if seed_checker_bypass:
+            logging.info("Bypassing Seed Checker")
+        else:
+            error_messages = self._assert_unique_seed_per_run(
+                result_files
+            )
+
+            if len(error_messages) > 0:
+                logging.error(" Seed checker failed and found the following errors: %s", '\n'.join(error_messages))
+                #print("Seed checker failed and found the following "
+                #      "errors:\n{}".format('\n'.join(error_messages)))
+                return False
         return True