Merge pull request #132 from mmoffatt2/master

Create Same Timestamp For Tensorboard Logs and Output Checkpoints

run_experiments.py

@@ -101,8 +101,10 @@ def run_command(config, config_basename, output_dir, csv_ckpt_dir, prefix, add_n
                 best_val_loss_from, override_max_iters, override_dataset, override_block_size):
     formatted_name = format_config_name(config, config_basename, prefix, add_names)
     config['tensorboard_run_name'] = formatted_name
-    config['out_dir'] = os.path.join(output_dir, f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_{formatted_name}")
-
+    timestamp_prefix = datetime.now().strftime('%Y%m%d_%H%M%S')
+    config['out_dir'] = os.path.join(output_dir, f"{timestamp_prefix}_{formatted_name}")
+    base_command.extend(["--timestamp", timestamp_prefix])
+
     if override_max_iters:
         config['max_iters'] = str(override_max_iters)
     if override_dataset:

train.py

@@ -169,6 +169,7 @@ def parse_args():
     # Logging args
     logging_group.add_argument('--log_project', default='out-test', type=str)
     logging_group.add_argument('--log_run_name', default='logs-test', type=str)
+    logging_group.add_argument('--timestamp', default='', type=str)
 
     # CSV logging
     logging_group.add_argument('--csv_log', default=True, action=argparse.BooleanOptionalAction)
@@ -305,6 +306,8 @@ def setup(self):
         self.raw_model = self.model.module if self.ddp else self.model
 
         timestamp_prefix = time.strftime("%Y%m%d-%H%M%S")
+        if self.args.timestamp:
+            timestamp_prefix = self.args.timestamp
 
         # Tensorboard
         if self.args.tensorboard_log: