Merge branch 'main' into main

kellyaa · May 16, 2024 · 756faf1 · 756faf1
2 parents 62bb076 + eba20f3
commit 756faf1
Showing 1 changed file with 10 additions and 6 deletions.
diff --git a/build/launch_training.py b/build/launch_training.py
@@ -143,12 +143,16 @@ def main():
                     export_path,
                 )
 
-                create_merged_model(
-                    checkpoint_models=full_checkpoint_dir,
-                    export_path=export_path,
-                    base_model=model_args.model_name_or_path,
-                    save_tokenizer=True,
-                )
+                # ensure checkpoint dir has correct files, important with multi-gpu tuning
+                if os.path.exists(
+                    os.path.join(full_checkpoint_dir, "adapter_config.json")
+                ):
+                    create_merged_model(
+                        checkpoint_models=full_checkpoint_dir,
+                        export_path=export_path,
+                        base_model=model_args.model_name_or_path,
+                        save_tokenizer=True,
+                    )
             except Exception as e:  # pylint: disable=broad-except
                 logging.error(traceback.format_exc())
                 write_termination_log(