From d91d4e345d9b5d6838797fbd4013cb6f3dd15f38 Mon Sep 17 00:00:00 2001 From: "fuzailpalnak@gmail.com" Date: Sun, 2 Jan 2022 20:41:21 +0100 Subject: [PATCH 1/4] video inference, config for data creation --- README.md | 10 ++--- ivu/conf.py | 38 ++++++++++++++--- ivu/dataset.py | 108 +++++++++++++++++++++++++++++++++++-------------- ivu/trainer.py | 10 +++-- ivu/utils.py | 61 ++++++++++++++++++++++++++++ 5 files changed, 183 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 3c025ac..0803ece 100644 --- a/README.md +++ b/README.md @@ -22,25 +22,25 @@ |... #### Generate Data From images - + ```python -from ivu.dataset import data_set_over_images +from ivu.dataset import training_data_set_over_images where_is_my_data = "where_is_my_data_path" where_i_want_to_store_my_data = "where_i_want_to_store_my_data_path" -data_set_over_images(where_is_my_data, where_i_want_to_store_my_data) +training_data_set_over_images(where_is_my_data, where_i_want_to_store_my_data) ``` #### Generate Data from Videos ```python -from ivu.dataset import data_set_over_videos +from ivu.dataset import training_data_set_over_videos where_is_my_data = "where_is_my_data_path" where_i_want_to_store_my_data = "where_i_want_to_store_my_data_path" # Set the width and height to a non negative integer if the frame has to be resized -data_set_over_videos(where_is_my_data, where_i_want_to_store_my_data, width=-1, height=-1) +training_data_set_over_videos(where_is_my_data, where_i_want_to_store_my_data, width=-1, height=-1) ``` #### Load generated data diff --git a/ivu/conf.py b/ivu/conf.py index 27afb69..483adc7 100644 --- a/ivu/conf.py +++ b/ivu/conf.py @@ -9,10 +9,21 @@ from ivu import models -class Config: - def __init__(self, pth: str): +class Conf: + def __init__(self, pth): self._config = OmegaConf.load(pth) + def get_entry(self, name: str): + return self._config[name] + + def get_sub_value_entry(self, name: str, sub_value: str): + return self._config[name][sub_value] + + +class TrainConf(Conf): + def __init__(self, pth: str): + super().__init__(pth) + self._optimizer = None self._callbacks = None self._loss_function = None @@ -20,9 +31,6 @@ def __init__(self, pth: str): self._log_dir = self._config.log_dir self._model_pth, self._graph_pth = self._create_log_dir() - def get_entry(self, name: str): - return self._config[name] - def get_loss(self): return getattr(losses, self._config.loss.name)(**self._config.loss.parameters) @@ -57,7 +65,27 @@ def _create_log_dir(self): ) +class DataConf(Conf): + def __init__(self, pth): + super().__init__(pth) + + def get_saved_model_pth(self): + return self._config.get_sub_value_entry("inference", "model_pth") + + def get_video_parameters(self): + return self._config.get_entry("video") + + def get_pose_estimators_parameters(self): + return self._config.get_entry("pose") + + def get_inference_parameters(self): + return self._config.get_entry("inference") + + # # conf = Config(r"config/normalized_sequence_distance_matrix.yaml") # conf.get_callbacks() # conf.get_entry("epochs") + +# conf = DataConf(r"/home/palnak/Workspace/Studium/courseWork/IVU/config/data.yaml") +# print(conf.get_sub_value_entry("inference", "model_pth")) diff --git a/ivu/dataset.py b/ivu/dataset.py index 88e4597..e8bee6b 100644 --- a/ivu/dataset.py +++ b/ivu/dataset.py @@ -1,19 +1,16 @@ import os from statistics import mode -from collections import defaultdict, Counter +from collections import defaultdict import cv2 import numpy as np from py_oneliner import one_liner -from scipy.spatial.distance import squareform, pdist from ivu.pose_estimator.base_pose_estimator import PoseEstimationFailedError from ivu.pose_estimator.media_pipe_estimator import get_media_pipe_pose_estimator -from ivu.pose_estimator.pose_landmarks import ( - Pose16LandmarksBodyModel, - landmarks_to_embedding, -) + + from ivu.utils import ( folder_generator, save_pickle, @@ -22,9 +19,77 @@ shuffle_two_list_together, one_hot, load_pickle, + get_pose_data_from_rgb_frame, + get_normalized_distance_matrix, ) +class VideoInferenceInputData: + def __init__( + self, + data_dir=None, + pose_estimator_complexity=1, + use_pose_estimator_over_static_image=True, + frame_width=-1, + frame_height=-1, + stride=128, + **kwargs, + ): + self._pose_estimator = get_media_pipe_pose_estimator( + complexity=pose_estimator_complexity, + static_image_mode=use_pose_estimator_over_static_image, + ) + self._data_dir = data_dir + self._frame_width = frame_width + self._frame_height = frame_height + self._stride = stride + + @staticmethod + def _adjust_frame_for_video_frame(x, stride): + n_samples = x.shape[0] + + indices = np.arange(n_samples) + end = min(0 + stride, n_samples) + stride_idx = indices[0 - stride + end - 0 : end] + return x[stride_idx] + + def data_for_normalized_distance_matrix(self): + + files = os.listdir(self._data_dir) + for iterator, file in enumerate(files): + input_data = list() + file_path = os.path.join(*[self._data_dir, file]) + vr = read_video( + file_path, width=self._frame_width, height=self._frame_height + ) + + for stride_iterator, frame in enumerate(range(len(vr))): + one_liner.one_line( + tag=f"PROGRESS [VIDEOS: {iterator + 1}/{len(files)}] [CURRENT FILE : {file}]", + tag_data=f"[FRAMES : {frame + 1}/{len(vr)}]", + to_reset_data=True, + tag_color="red", + tag_data_color="red", + ) + normalized_distance_matrix = get_normalized_distance_matrix( + vr[frame].asnumpy(), frame + ) + input_data.append( + normalized_distance_matrix[ + np.triu_indices(normalized_distance_matrix.shape[0], k=1) + ] + ) + if stride_iterator + 1 % self._stride == 0: + yield iterator, file, np.array(input_data) + elif ( + stride_iterator + 1 == len(vr) + and stride_iterator + 1 % self._stride != 0 + ): + yield iterator, file, self._adjust_frame_for_video_frame( + np.array(input_data), self._stride + ) + + class TrainInputData: def __init__(self, x, y): self._x = x @@ -112,26 +177,7 @@ def store(self, save_dir: str = None): save_pickle(self._meta, pth) -def get_pose_data_from_rgb_frame(frame, pose_estimator): - body_key_points = pose_estimator.get_key_points_from_image(frame) - distance_matrix = squareform(pdist(np.array(body_key_points))) - - normalized_body_key_points = landmarks_to_embedding( - body_key_points, Pose16LandmarksBodyModel - ) - normalized_distance_matrix = squareform( - pdist(np.array(normalized_body_key_points[0])) - ) - - return ( - body_key_points, - distance_matrix, - normalized_body_key_points, - normalized_distance_matrix, - ) - - -def generate_data_over_images( +def generate_training_data_over_images( data_set_dir: str, pose_estimator_complexity=1, use_pose_estimator_over_static_image=True, @@ -184,7 +230,7 @@ def generate_data_over_images( return data -def generate_data_over_videos( +def generate_training_data_over_videos( data_set_dir: str, pose_estimator_complexity=1, use_pose_estimator_over_static_image=True, @@ -245,13 +291,13 @@ def create_sequence_data_set_over_videos(data_set_path, frame_count): pass -def data_set_over_images( +def training_data_set_over_images( data_set_dir: str, save_dir: str, pose_estimator_complexity=1, use_pose_estimator_over_static_image=True, ): - data = generate_data_over_images( + data = generate_training_data_over_images( data_set_dir=data_set_dir, pose_estimator_complexity=pose_estimator_complexity, use_pose_estimator_over_static_image=use_pose_estimator_over_static_image, @@ -259,7 +305,7 @@ def data_set_over_images( data.store(save_dir=save_dir) -def data_set_over_videos( +def training_data_set_over_videos( data_set_dir: str, save_dir: str, pose_estimator_complexity=1, @@ -267,7 +313,7 @@ def data_set_over_videos( width=-1, height=-1, ): - data = generate_data_over_videos( + data = generate_training_data_over_videos( data_set_dir=data_set_dir, pose_estimator_complexity=pose_estimator_complexity, use_pose_estimator_over_static_image=use_pose_estimator_over_static_image, diff --git a/ivu/trainer.py b/ivu/trainer.py index 643f0b8..423fb40 100644 --- a/ivu/trainer.py +++ b/ivu/trainer.py @@ -1,4 +1,4 @@ -from ivu.conf import Config +from ivu.conf import TrainConf from ivu.dataset import TrainInputData @@ -10,7 +10,11 @@ def __init__(self, train_data, val_data, config): def start_training(self): model = self._config.get_model() - model.compile(optimizer=self._config.get_optimizer(), loss=self._config.get_loss(), metrics="accuracy") + model.compile( + optimizer=self._config.get_optimizer(), + loss=self._config.get_loss(), + metrics="accuracy", + ) _ = model.fit( self._train_data[0], @@ -32,7 +36,7 @@ def train_with_normalized_key_points(cls): @classmethod def train_with_normalized_distance_matrix(cls, data_pth, conf_pth): input_data = TrainInputData.data_with_normalized_distance_matrix(data_pth) - conf = Config(conf_pth) + conf = TrainConf(conf_pth) parameters = conf.get_entry("data") train_data, val_data = input_data.create_sequence_data( diff --git a/ivu/utils.py b/ivu/utils.py index 54dc902..0d92eb9 100644 --- a/ivu/utils.py +++ b/ivu/utils.py @@ -8,6 +8,12 @@ import pandas as pd import numpy as np from decord import VideoReader, cpu +from scipy.spatial.distance import squareform, pdist + +from ivu.pose_estimator.pose_landmarks import ( + landmarks_to_embedding, + Pose16LandmarksBodyModel, +) def shuffle_two_list_together(x, y): @@ -90,3 +96,58 @@ def read_video(pth, width=-1, height=-1) -> decord.VideoReader: def extract_from_data_frames(data_frame: pd.DataFrame, columns: List): return data_frame[columns] + + +def get_pose_data_from_rgb_frame(frame, pose_estimator): + body_key_points = pose_estimator.get_key_points_from_image(frame) + distance_matrix = squareform(pdist(np.array(body_key_points))) + + normalized_body_key_points = landmarks_to_embedding( + body_key_points, Pose16LandmarksBodyModel + ) + normalized_distance_matrix = squareform( + pdist(np.array(normalized_body_key_points[0])) + ) + + return ( + body_key_points, + distance_matrix, + normalized_body_key_points, + normalized_distance_matrix, + ) + + +def normalize_body_key_points(body_key_points, body_model): + return landmarks_to_embedding(body_key_points, body_model)[0] + + +def get_body_key_points(pose_estimator, rgb_input): + return pose_estimator.get_key_points_from_image(rgb_input) + + +def get_distance_matrix(pose_estimator, rgb_input): + body_key_points = pose_estimator.get_key_points_from_image(rgb_input) + return get_distance_matrix_from_key_points(np.array(body_key_points)) + + +def get_distance_matrix_from_key_points(body_key_points): + return squareform(pdist(np.array(body_key_points))) + + +def get_body_normalized_key_points(pose_estimator, rgb_input): + return normalize_body_key_points( + pose_estimator.get_key_points_from_image(rgb_input), Pose16LandmarksBodyModel + ) + + +def get_normalized_distance_matrix(pose_estimator, rgb_input): + body_key_points = pose_estimator.get_key_points_from_image(rgb_input) + return squareform( + pdist(normalize_body_key_points(body_key_points, Pose16LandmarksBodyModel)) + ) + + +def get_normalized_distance_matrix_from_body_key_points(body_key_points): + return squareform( + pdist(normalize_body_key_points(body_key_points, Pose16LandmarksBodyModel)) + ) From a6a173a3ce5034622473f2de04dcce316fd59bb1 Mon Sep 17 00:00:00 2001 From: "fuzailpalnak@gmail.com" Date: Fri, 7 Jan 2022 18:27:56 +0100 Subject: [PATCH 2/4] inference over videos --- .gitignore | 4 +++- config/data.yaml | 17 +++++++++++++++++ ivu/conf.py | 8 ++++---- ivu/dataset.py | 2 +- ivu/inference.py | 40 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 65 insertions(+), 6 deletions(-) create mode 100644 config/data.yaml create mode 100644 ivu/inference.py diff --git a/.gitignore b/.gitignore index a2de9ea..69c3159 100644 --- a/.gitignore +++ b/.gitignore @@ -130,4 +130,6 @@ dmypy.json datasets .idea/* -logs \ No newline at end of file +logs +trained_models +trained_model \ No newline at end of file diff --git a/config/data.yaml b/config/data.yaml new file mode 100644 index 0000000..8cea43e --- /dev/null +++ b/config/data.yaml @@ -0,0 +1,17 @@ +data_dir: +save_dir: + +pose: + pose_estimator_complexity: 1 + use_pose_estimator_over_static_image: True + +video: + frame_width: -1 + frame_height: -1 + stride: 300 + +inference: + model_pth: + infer_for: normalized_distance_matrix + + diff --git a/ivu/conf.py b/ivu/conf.py index 483adc7..6ffe730 100644 --- a/ivu/conf.py +++ b/ivu/conf.py @@ -70,16 +70,16 @@ def __init__(self, pth): super().__init__(pth) def get_saved_model_pth(self): - return self._config.get_sub_value_entry("inference", "model_pth") + return self.get_sub_value_entry("inference", "model_pth") def get_video_parameters(self): - return self._config.get_entry("video") + return self.get_entry("video") def get_pose_estimators_parameters(self): - return self._config.get_entry("pose") + return self.get_entry("pose") def get_inference_parameters(self): - return self._config.get_entry("inference") + return self.get_entry("inference") # diff --git a/ivu/dataset.py b/ivu/dataset.py index e8bee6b..044e0f0 100644 --- a/ivu/dataset.py +++ b/ivu/dataset.py @@ -72,7 +72,7 @@ def data_for_normalized_distance_matrix(self): tag_data_color="red", ) normalized_distance_matrix = get_normalized_distance_matrix( - vr[frame].asnumpy(), frame + pose_estimator=self._pose_estimator, rgb_input=vr[frame].asnumpy() ) input_data.append( normalized_distance_matrix[ diff --git a/ivu/inference.py b/ivu/inference.py new file mode 100644 index 0000000..8cb729c --- /dev/null +++ b/ivu/inference.py @@ -0,0 +1,40 @@ +import numpy as np +import tensorflow +from ivu.conf import DataConf +from ivu.dataset import VideoInferenceInputData + + +class Inference: + def __init__(self, model: tensorflow.keras.Model, conf: DataConf): + self._model = model + self._conf = conf + + @staticmethod + def _load_model(pth): + model = tensorflow.keras.models.load_model(pth, compile=True) + return model + + def run(self): + + pose_estimator_param = self._conf.get_pose_estimators_parameters() + video_param = self._conf.get_video_parameters() + inference_param = self._conf.get_inference_parameters() + + video_inference = VideoInferenceInputData( + data_dir=self._conf.get_entry("data_dir"), + **{**pose_estimator_param, **video_param, **inference_param} + ) + + for ( + file_iterator, + file, + input_data, + ) in video_inference.data_for_normalized_distance_matrix(): + input_data = np.expand_dims(input_data, axis=0) + prediction = self._model.predict(input_data) + print(tensorflow.argmax(prediction)) + + @classmethod + def init_inference_from_config(cls, pth): + conf = DataConf(pth) + return cls(cls._load_model(conf.get_saved_model_pth()), conf) From d6401cb08eed920ac0c3ce2012f99ed50d9e7b9f Mon Sep 17 00:00:00 2001 From: "fuzailpalnak@gmail.com" Date: Sat, 8 Jan 2022 12:06:13 +0100 Subject: [PATCH 3/4] [Refactor] - temp dir creation, traning with config, inference over videos, seperated data scripts --- README.md | 4 +- config/data.yaml | 6 +- .../normalized_sequence_distance_matrix.yaml | 6 +- ivu/conf.py | 17 +- ivu/data/__init__.py | 0 ivu/{dataset.py => data/creation.py} | 153 ++---------------- ivu/data/infer.py | 76 +++++++++ ivu/data/training.py | 80 +++++++++ ivu/inference.py | 9 +- ivu/trainer.py | 2 +- ivu/utils.py | 34 +++- 11 files changed, 221 insertions(+), 166 deletions(-) create mode 100644 ivu/data/__init__.py rename ivu/{dataset.py => data/creation.py} (57%) create mode 100644 ivu/data/infer.py create mode 100644 ivu/data/training.py diff --git a/README.md b/README.md index 0803ece..165fe9f 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ #### Generate Data From images ```python -from ivu.dataset import training_data_set_over_images +from ivu.data.creation import training_data_set_over_images where_is_my_data = "where_is_my_data_path" where_i_want_to_store_my_data = "where_i_want_to_store_my_data_path" @@ -34,7 +34,7 @@ training_data_set_over_images(where_is_my_data, where_i_want_to_store_my_data) #### Generate Data from Videos ```python -from ivu.dataset import training_data_set_over_videos +from ivu.data.creation import training_data_set_over_videos where_is_my_data = "where_is_my_data_path" where_i_want_to_store_my_data = "where_i_want_to_store_my_data_path" diff --git a/config/data.yaml b/config/data.yaml index 8cea43e..ba96d4d 100644 --- a/config/data.yaml +++ b/config/data.yaml @@ -1,4 +1,4 @@ -data_dir: +data_dir: /home/palnak/Workspace/Studium/workout_assitant/video_dataset/subset_2/Good save_dir: pose: @@ -8,10 +8,10 @@ pose: video: frame_width: -1 frame_height: -1 - stride: 300 + stride: 128 inference: - model_pth: + model_pth: /home/palnak/Workspace/Studium/courseWork/IVU/logs/20220107-194454/chk infer_for: normalized_distance_matrix diff --git a/config/normalized_sequence_distance_matrix.yaml b/config/normalized_sequence_distance_matrix.yaml index 765d7e1..128d6d7 100644 --- a/config/normalized_sequence_distance_matrix.yaml +++ b/config/normalized_sequence_distance_matrix.yaml @@ -1,6 +1,6 @@ data: validation_split: 0.2 - stride: 300 + stride: 128 optimizer: name: Adam @@ -18,7 +18,7 @@ callbacks: EarlyStopping: parameters: min_delta: 0.001 - patience: 5 + patience: 8 verbose: 1 ModelCheckpoint: @@ -40,4 +40,4 @@ model: input_features: 136 n_classes: 7 -log_dir : path_to_where_logs_will_be_stored \ No newline at end of file +log_dir : logs/ \ No newline at end of file diff --git a/ivu/conf.py b/ivu/conf.py index 6ffe730..d5e492a 100644 --- a/ivu/conf.py +++ b/ivu/conf.py @@ -7,11 +7,19 @@ from tensorflow.keras import callbacks from tensorflow.keras import losses from ivu import models +from ivu.utils import log_in_tmp_dir, log_in_current_dir class Conf: def __init__(self, pth): self._config = OmegaConf.load(pth) + if "log_dir" not in self._config.keys(): + log_dir = log_in_tmp_dir() + self._config["log_dir"] = log_dir + + elif self._config["log_dir"] is None: + log_dir = log_in_current_dir() + self._config["log_dir"] = log_dir def get_entry(self, name: str): return self._config[name] @@ -80,12 +88,3 @@ def get_pose_estimators_parameters(self): def get_inference_parameters(self): return self.get_entry("inference") - - -# -# conf = Config(r"config/normalized_sequence_distance_matrix.yaml") -# conf.get_callbacks() -# conf.get_entry("epochs") - -# conf = DataConf(r"/home/palnak/Workspace/Studium/courseWork/IVU/config/data.yaml") -# print(conf.get_sub_value_entry("inference", "model_pth")) diff --git a/ivu/data/__init__.py b/ivu/data/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ivu/dataset.py b/ivu/data/creation.py similarity index 57% rename from ivu/dataset.py rename to ivu/data/creation.py index 044e0f0..7d489d4 100644 --- a/ivu/dataset.py +++ b/ivu/data/creation.py @@ -1,5 +1,4 @@ import os -from statistics import mode from collections import defaultdict import cv2 @@ -7,6 +6,7 @@ from py_oneliner import one_liner +from ivu.conf import DataConf from ivu.pose_estimator.base_pose_estimator import PoseEstimationFailedError from ivu.pose_estimator.media_pipe_estimator import get_media_pipe_pose_estimator @@ -15,151 +15,10 @@ folder_generator, save_pickle, read_video, - train_val_split, - shuffle_two_list_together, - one_hot, - load_pickle, get_pose_data_from_rgb_frame, - get_normalized_distance_matrix, ) -class VideoInferenceInputData: - def __init__( - self, - data_dir=None, - pose_estimator_complexity=1, - use_pose_estimator_over_static_image=True, - frame_width=-1, - frame_height=-1, - stride=128, - **kwargs, - ): - self._pose_estimator = get_media_pipe_pose_estimator( - complexity=pose_estimator_complexity, - static_image_mode=use_pose_estimator_over_static_image, - ) - self._data_dir = data_dir - self._frame_width = frame_width - self._frame_height = frame_height - self._stride = stride - - @staticmethod - def _adjust_frame_for_video_frame(x, stride): - n_samples = x.shape[0] - - indices = np.arange(n_samples) - end = min(0 + stride, n_samples) - stride_idx = indices[0 - stride + end - 0 : end] - return x[stride_idx] - - def data_for_normalized_distance_matrix(self): - - files = os.listdir(self._data_dir) - for iterator, file in enumerate(files): - input_data = list() - file_path = os.path.join(*[self._data_dir, file]) - vr = read_video( - file_path, width=self._frame_width, height=self._frame_height - ) - - for stride_iterator, frame in enumerate(range(len(vr))): - one_liner.one_line( - tag=f"PROGRESS [VIDEOS: {iterator + 1}/{len(files)}] [CURRENT FILE : {file}]", - tag_data=f"[FRAMES : {frame + 1}/{len(vr)}]", - to_reset_data=True, - tag_color="red", - tag_data_color="red", - ) - normalized_distance_matrix = get_normalized_distance_matrix( - pose_estimator=self._pose_estimator, rgb_input=vr[frame].asnumpy() - ) - input_data.append( - normalized_distance_matrix[ - np.triu_indices(normalized_distance_matrix.shape[0], k=1) - ] - ) - if stride_iterator + 1 % self._stride == 0: - yield iterator, file, np.array(input_data) - elif ( - stride_iterator + 1 == len(vr) - and stride_iterator + 1 % self._stride != 0 - ): - yield iterator, file, self._adjust_frame_for_video_frame( - np.array(input_data), self._stride - ) - - -class TrainInputData: - def __init__(self, x, y): - self._x = x - self._y = y - - def create_sequence_data(self, validation_split: float = None, stride: int = 300): - _x = list() - _y = list() - - n_samples = self._x.shape[0] - indices = np.arange(n_samples) - - for start in range(0, n_samples, stride): - end = min(start + stride, n_samples) - - batch_idx = indices[start:end] - if len(batch_idx) < stride: - batch_idx = indices[start - stride + end - start : end] - - _x.append(self._x[batch_idx]) - _y.append(mode(self._y[batch_idx])) - # _y.append(Counter(y[batch_idx]).most_common(1)[0][0]) - - _x_shuffled, _y_shuffled = shuffle_two_list_together(_x, _y) - if validation_split is not None: - _x_train, _y_train, _x_val, _y_val = train_val_split( - np.array(_x_shuffled), np.array(_y_shuffled) - ) - else: - _x_train, _y_train, _x_val, _y_val = ( - np.array(_x_shuffled), - np.array(_y_shuffled), - None, - None, - ) - return ( - _x_train, - one_hot(_y_train), - ), None if _x_val is None or _y_val is None else (_x_val, one_hot(_y_val)) - - @classmethod - def data_with_normalized_key_points(cls, pth): - pass - - @classmethod - def data_with_normalized_distance_matrix(cls, pth): - df = load_pickle(pth) - subset = df[ - [ - "normalized_distance_matrix", - "class_label", - "class_label_index", - "frame_details", - "frame_number", - ] - ] - x = np.array(subset["normalized_distance_matrix"].tolist()) - y = np.array(subset["class_label_index"].tolist()) - - return cls(x, y) - - @classmethod - def data_with_distance_matrix(cls, pth): - pass - - @classmethod - def data_with_key_points(cls, pth): - pass - - class GeneratedData: def __init__(self): self._meta = defaultdict(list) @@ -321,3 +180,13 @@ def training_data_set_over_videos( frame_height=height, ) data.store(save_dir=save_dir) + + +def training_data_set_over_videos_using_conf(pth): + conf = DataConf(pth) + training_data_set_over_videos( + data_set_dir=conf.get_entry("data_dir"), + save_dir=conf.get_entry("save_dir"), + **conf.get_pose_estimators_parameters(), + **conf.get_video_parameters(), + ) diff --git a/ivu/data/infer.py b/ivu/data/infer.py new file mode 100644 index 0000000..c87e8fb --- /dev/null +++ b/ivu/data/infer.py @@ -0,0 +1,76 @@ +import os +import numpy as np + +from py_oneliner import one_liner + +from ivu.utils import ( + read_video, + get_normalized_distance_matrix, +) +from ivu.pose_estimator.media_pipe_estimator import get_media_pipe_pose_estimator + + +class VideoInferenceInputData: + def __init__( + self, + data_dir=None, + pose_estimator_complexity=1, + use_pose_estimator_over_static_image=True, + frame_width=-1, + frame_height=-1, + stride=128, + **kwargs, + ): + self._pose_estimator = get_media_pipe_pose_estimator( + complexity=pose_estimator_complexity, + static_image_mode=use_pose_estimator_over_static_image, + ) + self._data_dir = data_dir + self._frame_width = frame_width + self._frame_height = frame_height + self._stride = stride + + @staticmethod + def _adjust_frame_for_video_frame(x, stride): + n_samples = x.shape[0] + + indices = np.arange(n_samples) + end = min(0 + stride, n_samples) + stride_idx = indices[0 - stride + end - 0 : end] + return x[stride_idx] + + def data_for_normalized_distance_matrix(self): + + files = os.listdir(self._data_dir) + for iterator, file in enumerate(files): + input_data = list() + file_path = os.path.join(*[self._data_dir, file]) + vr = read_video( + file_path, width=self._frame_width, height=self._frame_height + ) + + for stride_iterator, frame in enumerate(range(len(vr))): + one_liner.one_line( + tag=f"PROGRESS [VIDEOS: {iterator + 1}/{len(files)}] [CURRENT FILE : {file}]", + tag_data=f"[FRAMES : {frame + 1}/{len(vr)}]", + to_reset_data=True, + tag_color="red", + tag_data_color="red", + ) + normalized_distance_matrix = get_normalized_distance_matrix( + pose_estimator=self._pose_estimator, rgb_input=vr[frame].asnumpy() + ) + input_data.append( + normalized_distance_matrix[ + np.triu_indices(normalized_distance_matrix.shape[0], k=1) + ] + ) + if (stride_iterator + 1) % self._stride == 0: + yield iterator, file, frame, np.array(input_data) + elif ( + stride_iterator + 1 == len(vr) + and (stride_iterator + 1) % self._stride != 0 + ): + yield iterator, file, frame, self._adjust_frame_for_video_frame( + np.array(input_data), self._stride + ) diff --git a/ivu/data/training.py b/ivu/data/training.py new file mode 100644 index 0000000..8b78243 --- /dev/null +++ b/ivu/data/training.py @@ -0,0 +1,80 @@ +from statistics import mode +import numpy as np + + +from ivu.utils import ( + train_val_split, + shuffle_two_list_together, + one_hot, + load_pickle, +) + + +class TrainInputData: + def __init__(self, x, y): + self._x = x + self._y = y + + def create_sequence_data(self, validation_split: float = None, stride: int = 300): + _x = list() + _y = list() + + n_samples = self._x.shape[0] + indices = np.arange(n_samples) + + for start in range(0, n_samples, stride): + end = min(start + stride, n_samples) + + batch_idx = indices[start:end] + if len(batch_idx) < stride: + batch_idx = indices[start - stride + end - start : end] + + _x.append(self._x[batch_idx]) + _y.append(mode(self._y[batch_idx])) + # _y.append(Counter(y[batch_idx]).most_common(1)[0][0]) + + _x_shuffled, _y_shuffled = shuffle_two_list_together(_x, _y) + if validation_split is not None: + _x_train, _y_train, _x_val, _y_val = train_val_split( + np.array(_x_shuffled), np.array(_y_shuffled) + ) + else: + _x_train, _y_train, _x_val, _y_val = ( + np.array(_x_shuffled), + np.array(_y_shuffled), + None, + None, + ) + return ( + _x_train, + one_hot(_y_train), + ), None if _x_val is None or _y_val is None else (_x_val, one_hot(_y_val)) + + @classmethod + def data_with_normalized_key_points(cls, pth): + pass + + @classmethod + def data_with_normalized_distance_matrix(cls, pth): + df = load_pickle(pth) + subset = df[ + [ + "normalized_distance_matrix", + "class_label", + "class_label_index", + "frame_details", + "frame_number", + ] + ] + x = np.array(subset["normalized_distance_matrix"].tolist()) + y = np.array(subset["class_label_index"].tolist()) + + return cls(x, y) + + @classmethod + def data_with_distance_matrix(cls, pth): + pass + + @classmethod + def data_with_key_points(cls, pth): + pass diff --git a/ivu/inference.py b/ivu/inference.py index 8cb729c..6d70d64 100644 --- a/ivu/inference.py +++ b/ivu/inference.py @@ -1,7 +1,9 @@ +from collections import defaultdict + import numpy as np import tensorflow from ivu.conf import DataConf -from ivu.dataset import VideoInferenceInputData +from ivu.data.infer import VideoInferenceInputData class Inference: @@ -25,14 +27,17 @@ def run(self): **{**pose_estimator_param, **video_param, **inference_param} ) + predictions = defaultdict(list) for ( file_iterator, file, + frame, input_data, ) in video_inference.data_for_normalized_distance_matrix(): input_data = np.expand_dims(input_data, axis=0) prediction = self._model.predict(input_data) - print(tensorflow.argmax(prediction)) + predictions[file].append(np.argmax(prediction)) + print(np.argmax(prediction)) @classmethod def init_inference_from_config(cls, pth): diff --git a/ivu/trainer.py b/ivu/trainer.py index 423fb40..dbf5aea 100644 --- a/ivu/trainer.py +++ b/ivu/trainer.py @@ -1,5 +1,5 @@ from ivu.conf import TrainConf -from ivu.dataset import TrainInputData +from ivu.data.training import TrainInputData class Trainer: diff --git a/ivu/utils.py b/ivu/utils.py index 0d92eb9..6a62d8d 100644 --- a/ivu/utils.py +++ b/ivu/utils.py @@ -1,6 +1,7 @@ import pickle import os import random +import tempfile from collections import defaultdict from typing import List @@ -16,6 +17,33 @@ ) +def log_in_tmp_dir(): + # https://stackoverflow.com/questions/847850/cross-platform-way-of-getting-temp-directory-in-python + log_dir = os.path.join(tempfile.gettempdir(), "ivu_logs") + if not os.path.exists: + os.makedirs(log_dir) + return log_dir + + +def log_in_current_dir(): + log_dir = os.path.join(os.getcwd(), "ivu_logs") + if not os.path.exists: + os.makedirs(log_dir) + return log_dir + + +def get_class_association(df): + class_ass = dict() + for i in range(0, 7): + d = filter_data_frame(df, "class_label_index", i)["class_label"] + class_ass[i] = np.unique(d.to_numpy())[0] + return class_ass + + +def filter_data_frame(df: pd.DataFrame, name, value): + return df.loc[df[name] == value] + + def shuffle_two_list_together(x, y): # https://stackoverflow.com/a/23289591 c = list(zip(x, y)) @@ -141,10 +169,8 @@ def get_body_normalized_key_points(pose_estimator, rgb_input): def get_normalized_distance_matrix(pose_estimator, rgb_input): - body_key_points = pose_estimator.get_key_points_from_image(rgb_input) - return squareform( - pdist(normalize_body_key_points(body_key_points, Pose16LandmarksBodyModel)) - ) + body_key_points = get_body_normalized_key_points(pose_estimator, rgb_input) + return get_distance_matrix_from_key_points(body_key_points) def get_normalized_distance_matrix_from_body_key_points(body_key_points): From 1b5e152155a572c6d22c0b175ba41eeee6275d6a Mon Sep 17 00:00:00 2001 From: "fuzailpalnak@gmail.com" Date: Sat, 8 Jan 2022 12:09:35 +0100 Subject: [PATCH 4/4] readme updated --- README.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 165fe9f..8432e32 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,7 @@ where_i_want_to_store_my_data = "where_i_want_to_store_my_data_path" training_data_set_over_images(where_is_my_data, where_i_want_to_store_my_data) ``` -#### Generate Data from Videos - +#### Generate Data from Videos with Parameters ```python from ivu.data.creation import training_data_set_over_videos @@ -42,6 +41,13 @@ where_i_want_to_store_my_data = "where_i_want_to_store_my_data_path" # Set the width and height to a non negative integer if the frame has to be resized training_data_set_over_videos(where_is_my_data, where_i_want_to_store_my_data, width=-1, height=-1) ``` + +#### Generate Data from Videos with config +```python +from ivu.data.creation import training_data_set_over_videos_using_conf +training_data_set_over_videos_using_conf("path_to_dat_config") +``` + #### Load generated data @@ -70,3 +76,12 @@ trainer = Trainer.train_with_normalized_distance_matrix(data_pth=r"path_to_pickl conf_pth=r"path_to_config") trainer.start_training() ``` + +#### Inference on data +```python +from ivu.inference import Inference + +vd = Inference.init_inference_from_config("path_to_config_file") +vd.run() + +```