From 0810ed581d139668f50dd90a378fffd1b8dfa69f Mon Sep 17 00:00:00 2001 From: Matt Whiteway Date: Wed, 11 Dec 2024 15:56:31 +0000 Subject: [PATCH] start post-training analysis --- .gitignore | 2 ++ configs/pipeline.yaml | 4 +-- lp3d_analysis/train.py | 29 +++++++++++++++--- lp3d_analysis/utils.py | 22 +++++++++++++ pipelines/pipeline_simple.py | 11 +++++++ ...events.1733863018.ip-10-192-11-211.42209.2 | Bin 1067 -> 0 bytes 6 files changed, 61 insertions(+), 7 deletions(-) create mode 100644 lp3d_analysis/utils.py delete mode 100644 tb_logs/test_model/version_0/events.out.tfevents.1733863018.ip-10-192-11-211.42209.2 diff --git a/.gitignore b/.gitignore index aedc8d7..e313e8b 100644 --- a/.gitignore +++ b/.gitignore @@ -81,3 +81,5 @@ venv/ # written by setuptools_scm **/_version.py + +tb_logs/ \ No newline at end of file diff --git a/configs/pipeline.yaml b/configs/pipeline.yaml index c005c11..d517ee7 100644 --- a/configs/pipeline.yaml +++ b/configs/pipeline.yaml @@ -32,8 +32,8 @@ train_networks: # run inference on videos in these directories (independent of training videos) inference_dirs: # - videos_new - # - videos-for-each-labeled-frame - - videos_debug + - videos-for-each-labeled-frame + # - videos_debug # training parameters # min_steps: 5000 # max_steps: 5000 diff --git a/lp3d_analysis/train.py b/lp3d_analysis/train.py index 815bba7..730b3eb 100644 --- a/lp3d_analysis/train.py +++ b/lp3d_analysis/train.py @@ -19,8 +19,14 @@ from lightning_pose.utils.predictions import (predict_dataset, predict_single_video) from lightning_pose.utils.scripts import ( # get_callbacks, - calculate_train_batches, compute_metrics, get_data_module, get_dataset, - get_imgaug_transform, get_loss_factories, get_model) + calculate_train_batches, + compute_metrics, + get_data_module, + get_dataset, + get_imgaug_transform, + get_loss_factories, + get_model, +) from moviepy.editor import VideoFileClip from omegaconf import DictConfig, OmegaConf from typeguard import typechecked @@ -420,11 +426,24 @@ def train_and_infer( # Run inference on all InD/OOD videos and compute unsupervised metrics for video_dir in inference_dirs: - video_files = \ - [f for f in os.listdir(os.path.join(data_dir, video_dir)) if f.endswith('.mp4')] + + video_files = [ + f for f in os.listdir(os.path.join(data_dir, video_dir)) if f.endswith('.mp4') + ] + + if len(video_files) == 0: + # assume we have a nested directory: directories full of mp4 files + video_files = [] + sub_video_dirs = os.listdir(os.path.join(data_dir, video_dir)) + for sub_video_dir in sub_video_dirs: + sub_video_dir_abs = os.path.join(data_dir, video_dir, sub_video_dir) + files_tmp = os.listdir(sub_video_dir_abs) + video_files += [f'{sub_video_dir}/{f}' for f in files_tmp if f.endswith('.mp4')] + for video_file in video_files: if csv_prefix: - inference_csv_name = f'{csv_prefix}_{video_file.replace(".mp4", ".csv")}' + raise NotImplementedError + # inference_csv_name = f'{csv_prefix}_{video_file.replace(".mp4", ".csv")}' else: inference_csv_name = video_file.replace(".mp4", ".csv") inference_csv = os.path.join(results_dir, video_dir, inference_csv_name) diff --git a/lp3d_analysis/utils.py b/lp3d_analysis/utils.py new file mode 100644 index 0000000..3dea6e3 --- /dev/null +++ b/lp3d_analysis/utils.py @@ -0,0 +1,22 @@ + + +def extract_ood_frame_predictions( + data_dir: str, + results_dir: str, + overwrite: bool, +) -> None: + + pass + + # look for all files that end in _new.csv -> these are OOD labels + # loop through these + # for each, load the csv file, and iterate through the rows/index + # 'labeled-data//img<#>.png' + # s = 'labeled-data/vid_name/img0000.png' + # s2 = '/'.join(s.split('/')[1:]) + # s3 = s2.replace('png', 'mp4') + # load 51-frame csv file + # extract center frame + # put in dataframe + # save out predictions_.csv + # compute pixel diff --git a/pipelines/pipeline_simple.py b/pipelines/pipeline_simple.py index c3be9d4..de6d587 100644 --- a/pipelines/pipeline_simple.py +++ b/pipelines/pipeline_simple.py @@ -3,16 +3,21 @@ from lp3d_analysis.io import load_cfgs from lp3d_analysis.train import train_and_infer +from lp3d_analysis.utils import extract_ood_frame_predictions # TODO # - before train_and_infer, will be nice to put cfg updates in their own function +# - remove get_callbacks from lp3d_analysis.train.py +# - replace train function with one from LP +# - faster inference with OOD videos: don't rebuild the model every time VALID_MODEL_TYPES = [ 'supervised', 'context', ] + def pipeline(config_file: str): # ------------------------------------------- @@ -69,6 +74,12 @@ def pipeline(config_file: str): inference_dirs=cfg_pipe.train_networks.inference_dirs, overwrite=cfg_pipe.train_networks.overwrite, ) + # Clean up/reorganize OOD data + extract_ood_frame_predictions( + data_dir=data_dir, + results_dir=results_dir, + overwrite=cfg_pipe.train_networks.overwrite, + ) # # # # ------------------------------------------------------------------------------------- # # # # Post-process network outputs to generate potential pseudo labels (chosen in next step) diff --git a/tb_logs/test_model/version_0/events.out.tfevents.1733863018.ip-10-192-11-211.42209.2 b/tb_logs/test_model/version_0/events.out.tfevents.1733863018.ip-10-192-11-211.42209.2 deleted file mode 100644 index c57c669731d2aefb14bb07c2e9bda8860e9f9e71..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1067 zcmb7?K}Zx)7{}KcA?MR=-663Cm1TuUTXxsgJe1%T1mY>YhCJWSe!Dxk^JaMOO{S-A z9XbS^qOgk;Dbyi}j$JzmbqEZiK!^?<3_5fu>20&lC`r*A9?bXszxRFb_y1-R;kS2Z zX4Sa6vwm;n?e;}0S>e=S`ODXk$@8VTJme%Y;qi(`c{MKr*QI?9~+n?q;LUsMt*{_ED(iXqg zU`vA&q~TLeT|o+X-AC@Ys@v>y4W4Omter9+Y37=iO+%6eE?LB7-*cE089keV?p8=r zUdP{o9_Xr2d$Hm%%yeY?AYJvAn4qq&rms<3dR$c<2k^NXQ8B=(`tOAvFOYs9iD*#2 zRj3q7YBz|$eMOTKVK=ZEG?6Ps7KRQ+Ncf0?t1tQwPnZA*|pDjl#I zM_fU&{}4hYLEp0*;pmP_>qvyNf*E}@3EgH)e8c{B^L-n3b#tfF*om{l>?Cnm&6|Cq!-gJ1f1q3Ubr?DVp=qSs_LpHe25$CSeH;Ztd9Fw(Q*$;6^~Z{f zaPp8%QTbFEti$L{u)5BSmvWiOi4hoto`ZbMq}A;PP3Z`Uev`xLztD|Ze4K{