diff --git a/.gitignore b/.gitignore index a2de9ea..69c3159 100644 --- a/.gitignore +++ b/.gitignore @@ -130,4 +130,6 @@ dmypy.json datasets .idea/* -logs \ No newline at end of file +logs +trained_models +trained_model \ No newline at end of file diff --git a/README.md b/README.md index 3c025ac..8432e32 100644 --- a/README.md +++ b/README.md @@ -22,26 +22,32 @@ |... #### Generate Data From images - + ```python -from ivu.dataset import data_set_over_images +from ivu.data.creation import training_data_set_over_images where_is_my_data = "where_is_my_data_path" where_i_want_to_store_my_data = "where_i_want_to_store_my_data_path" -data_set_over_images(where_is_my_data, where_i_want_to_store_my_data) +training_data_set_over_images(where_is_my_data, where_i_want_to_store_my_data) ``` -#### Generate Data from Videos - +#### Generate Data from Videos with Parameters ```python -from ivu.dataset import data_set_over_videos +from ivu.data.creation import training_data_set_over_videos where_is_my_data = "where_is_my_data_path" where_i_want_to_store_my_data = "where_i_want_to_store_my_data_path" # Set the width and height to a non negative integer if the frame has to be resized -data_set_over_videos(where_is_my_data, where_i_want_to_store_my_data, width=-1, height=-1) +training_data_set_over_videos(where_is_my_data, where_i_want_to_store_my_data, width=-1, height=-1) +``` + +#### Generate Data from Videos with config +```python +from ivu.data.creation import training_data_set_over_videos_using_conf +training_data_set_over_videos_using_conf("path_to_dat_config") ``` + #### Load generated data @@ -70,3 +76,12 @@ trainer = Trainer.train_with_normalized_distance_matrix(data_pth=r"path_to_pickl conf_pth=r"path_to_config") trainer.start_training() ``` + +#### Inference on data +```python +from ivu.inference import Inference + +vd = Inference.init_inference_from_config("path_to_config_file") +vd.run() + +``` diff --git a/config/data.yaml b/config/data.yaml new file mode 100644 index 0000000..ba96d4d --- /dev/null +++ b/config/data.yaml @@ -0,0 +1,17 @@ +data_dir: /home/palnak/Workspace/Studium/workout_assitant/video_dataset/subset_2/Good +save_dir: + +pose: + pose_estimator_complexity: 1 + use_pose_estimator_over_static_image: True + +video: + frame_width: -1 + frame_height: -1 + stride: 128 + +inference: + model_pth: /home/palnak/Workspace/Studium/courseWork/IVU/logs/20220107-194454/chk + infer_for: normalized_distance_matrix + + diff --git a/config/normalized_sequence_distance_matrix.yaml b/config/normalized_sequence_distance_matrix.yaml index 765d7e1..128d6d7 100644 --- a/config/normalized_sequence_distance_matrix.yaml +++ b/config/normalized_sequence_distance_matrix.yaml @@ -1,6 +1,6 @@ data: validation_split: 0.2 - stride: 300 + stride: 128 optimizer: name: Adam @@ -18,7 +18,7 @@ callbacks: EarlyStopping: parameters: min_delta: 0.001 - patience: 5 + patience: 8 verbose: 1 ModelCheckpoint: @@ -40,4 +40,4 @@ model: input_features: 136 n_classes: 7 -log_dir : path_to_where_logs_will_be_stored \ No newline at end of file +log_dir : logs/ \ No newline at end of file diff --git a/ivu/conf.py b/ivu/conf.py index 27afb69..d5e492a 100644 --- a/ivu/conf.py +++ b/ivu/conf.py @@ -7,11 +7,30 @@ from tensorflow.keras import callbacks from tensorflow.keras import losses from ivu import models +from ivu.utils import log_in_tmp_dir, log_in_current_dir -class Config: - def __init__(self, pth: str): +class Conf: + def __init__(self, pth): self._config = OmegaConf.load(pth) + if "log_dir" not in self._config.keys(): + log_dir = log_in_tmp_dir() + self._config["log_dir"] = log_dir + + elif self._config["log_dir"] is None: + log_dir = log_in_current_dir() + self._config["log_dir"] = log_dir + + def get_entry(self, name: str): + return self._config[name] + + def get_sub_value_entry(self, name: str, sub_value: str): + return self._config[name][sub_value] + + +class TrainConf(Conf): + def __init__(self, pth: str): + super().__init__(pth) self._optimizer = None self._callbacks = None @@ -20,9 +39,6 @@ def __init__(self, pth: str): self._log_dir = self._config.log_dir self._model_pth, self._graph_pth = self._create_log_dir() - def get_entry(self, name: str): - return self._config[name] - def get_loss(self): return getattr(losses, self._config.loss.name)(**self._config.loss.parameters) @@ -57,7 +73,18 @@ def _create_log_dir(self): ) -# -# conf = Config(r"config/normalized_sequence_distance_matrix.yaml") -# conf.get_callbacks() -# conf.get_entry("epochs") +class DataConf(Conf): + def __init__(self, pth): + super().__init__(pth) + + def get_saved_model_pth(self): + return self.get_sub_value_entry("inference", "model_pth") + + def get_video_parameters(self): + return self.get_entry("video") + + def get_pose_estimators_parameters(self): + return self.get_entry("pose") + + def get_inference_parameters(self): + return self.get_entry("inference") diff --git a/ivu/data/__init__.py b/ivu/data/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ivu/dataset.py b/ivu/data/creation.py similarity index 65% rename from ivu/dataset.py rename to ivu/data/creation.py index 88e4597..7d489d4 100644 --- a/ivu/dataset.py +++ b/ivu/data/creation.py @@ -1,100 +1,24 @@ import os -from statistics import mode -from collections import defaultdict, Counter +from collections import defaultdict import cv2 import numpy as np from py_oneliner import one_liner -from scipy.spatial.distance import squareform, pdist +from ivu.conf import DataConf from ivu.pose_estimator.base_pose_estimator import PoseEstimationFailedError from ivu.pose_estimator.media_pipe_estimator import get_media_pipe_pose_estimator -from ivu.pose_estimator.pose_landmarks import ( - Pose16LandmarksBodyModel, - landmarks_to_embedding, -) + + from ivu.utils import ( folder_generator, save_pickle, read_video, - train_val_split, - shuffle_two_list_together, - one_hot, - load_pickle, + get_pose_data_from_rgb_frame, ) -class TrainInputData: - def __init__(self, x, y): - self._x = x - self._y = y - - def create_sequence_data(self, validation_split: float = None, stride: int = 300): - _x = list() - _y = list() - - n_samples = self._x.shape[0] - indices = np.arange(n_samples) - - for start in range(0, n_samples, stride): - end = min(start + stride, n_samples) - - batch_idx = indices[start:end] - if len(batch_idx) < stride: - batch_idx = indices[start - stride + end - start : end] - - _x.append(self._x[batch_idx]) - _y.append(mode(self._y[batch_idx])) - # _y.append(Counter(y[batch_idx]).most_common(1)[0][0]) - - _x_shuffled, _y_shuffled = shuffle_two_list_together(_x, _y) - if validation_split is not None: - _x_train, _y_train, _x_val, _y_val = train_val_split( - np.array(_x_shuffled), np.array(_y_shuffled) - ) - else: - _x_train, _y_train, _x_val, _y_val = ( - np.array(_x_shuffled), - np.array(_y_shuffled), - None, - None, - ) - return ( - _x_train, - one_hot(_y_train), - ), None if _x_val is None or _y_val is None else (_x_val, one_hot(_y_val)) - - @classmethod - def data_with_normalized_key_points(cls, pth): - pass - - @classmethod - def data_with_normalized_distance_matrix(cls, pth): - df = load_pickle(pth) - subset = df[ - [ - "normalized_distance_matrix", - "class_label", - "class_label_index", - "frame_details", - "frame_number", - ] - ] - x = np.array(subset["normalized_distance_matrix"].tolist()) - y = np.array(subset["class_label_index"].tolist()) - - return cls(x, y) - - @classmethod - def data_with_distance_matrix(cls, pth): - pass - - @classmethod - def data_with_key_points(cls, pth): - pass - - class GeneratedData: def __init__(self): self._meta = defaultdict(list) @@ -112,26 +36,7 @@ def store(self, save_dir: str = None): save_pickle(self._meta, pth) -def get_pose_data_from_rgb_frame(frame, pose_estimator): - body_key_points = pose_estimator.get_key_points_from_image(frame) - distance_matrix = squareform(pdist(np.array(body_key_points))) - - normalized_body_key_points = landmarks_to_embedding( - body_key_points, Pose16LandmarksBodyModel - ) - normalized_distance_matrix = squareform( - pdist(np.array(normalized_body_key_points[0])) - ) - - return ( - body_key_points, - distance_matrix, - normalized_body_key_points, - normalized_distance_matrix, - ) - - -def generate_data_over_images( +def generate_training_data_over_images( data_set_dir: str, pose_estimator_complexity=1, use_pose_estimator_over_static_image=True, @@ -184,7 +89,7 @@ def generate_data_over_images( return data -def generate_data_over_videos( +def generate_training_data_over_videos( data_set_dir: str, pose_estimator_complexity=1, use_pose_estimator_over_static_image=True, @@ -245,13 +150,13 @@ def create_sequence_data_set_over_videos(data_set_path, frame_count): pass -def data_set_over_images( +def training_data_set_over_images( data_set_dir: str, save_dir: str, pose_estimator_complexity=1, use_pose_estimator_over_static_image=True, ): - data = generate_data_over_images( + data = generate_training_data_over_images( data_set_dir=data_set_dir, pose_estimator_complexity=pose_estimator_complexity, use_pose_estimator_over_static_image=use_pose_estimator_over_static_image, @@ -259,7 +164,7 @@ def data_set_over_images( data.store(save_dir=save_dir) -def data_set_over_videos( +def training_data_set_over_videos( data_set_dir: str, save_dir: str, pose_estimator_complexity=1, @@ -267,7 +172,7 @@ def data_set_over_videos( width=-1, height=-1, ): - data = generate_data_over_videos( + data = generate_training_data_over_videos( data_set_dir=data_set_dir, pose_estimator_complexity=pose_estimator_complexity, use_pose_estimator_over_static_image=use_pose_estimator_over_static_image, @@ -275,3 +180,13 @@ def data_set_over_videos( frame_height=height, ) data.store(save_dir=save_dir) + + +def training_data_set_over_videos_using_conf(pth): + conf = DataConf(pth) + training_data_set_over_videos( + data_set_dir=conf.get_entry("data_dir"), + save_dir=conf.get_entry("save_dir"), + **conf.get_pose_estimators_parameters(), + **conf.get_video_parameters(), + ) diff --git a/ivu/data/infer.py b/ivu/data/infer.py new file mode 100644 index 0000000..c87e8fb --- /dev/null +++ b/ivu/data/infer.py @@ -0,0 +1,76 @@ +import os +import numpy as np + +from py_oneliner import one_liner + +from ivu.utils import ( + read_video, + get_normalized_distance_matrix, +) +from ivu.pose_estimator.media_pipe_estimator import get_media_pipe_pose_estimator + + +class VideoInferenceInputData: + def __init__( + self, + data_dir=None, + pose_estimator_complexity=1, + use_pose_estimator_over_static_image=True, + frame_width=-1, + frame_height=-1, + stride=128, + **kwargs, + ): + self._pose_estimator = get_media_pipe_pose_estimator( + complexity=pose_estimator_complexity, + static_image_mode=use_pose_estimator_over_static_image, + ) + self._data_dir = data_dir + self._frame_width = frame_width + self._frame_height = frame_height + self._stride = stride + + @staticmethod + def _adjust_frame_for_video_frame(x, stride): + n_samples = x.shape[0] + + indices = np.arange(n_samples) + end = min(0 + stride, n_samples) + stride_idx = indices[0 - stride + end - 0 : end] + return x[stride_idx] + + def data_for_normalized_distance_matrix(self): + + files = os.listdir(self._data_dir) + for iterator, file in enumerate(files): + input_data = list() + file_path = os.path.join(*[self._data_dir, file]) + vr = read_video( + file_path, width=self._frame_width, height=self._frame_height + ) + + for stride_iterator, frame in enumerate(range(len(vr))): + one_liner.one_line( + tag=f"PROGRESS [VIDEOS: {iterator + 1}/{len(files)}] [CURRENT FILE : {file}]", + tag_data=f"[FRAMES : {frame + 1}/{len(vr)}]", + to_reset_data=True, + tag_color="red", + tag_data_color="red", + ) + normalized_distance_matrix = get_normalized_distance_matrix( + pose_estimator=self._pose_estimator, rgb_input=vr[frame].asnumpy() + ) + input_data.append( + normalized_distance_matrix[ + np.triu_indices(normalized_distance_matrix.shape[0], k=1) + ] + ) + if (stride_iterator + 1) % self._stride == 0: + yield iterator, file, frame, np.array(input_data) + elif ( + stride_iterator + 1 == len(vr) + and (stride_iterator + 1) % self._stride != 0 + ): + yield iterator, file, frame, self._adjust_frame_for_video_frame( + np.array(input_data), self._stride + ) diff --git a/ivu/data/training.py b/ivu/data/training.py new file mode 100644 index 0000000..8b78243 --- /dev/null +++ b/ivu/data/training.py @@ -0,0 +1,80 @@ +from statistics import mode +import numpy as np + + +from ivu.utils import ( + train_val_split, + shuffle_two_list_together, + one_hot, + load_pickle, +) + + +class TrainInputData: + def __init__(self, x, y): + self._x = x + self._y = y + + def create_sequence_data(self, validation_split: float = None, stride: int = 300): + _x = list() + _y = list() + + n_samples = self._x.shape[0] + indices = np.arange(n_samples) + + for start in range(0, n_samples, stride): + end = min(start + stride, n_samples) + + batch_idx = indices[start:end] + if len(batch_idx) < stride: + batch_idx = indices[start - stride + end - start : end] + + _x.append(self._x[batch_idx]) + _y.append(mode(self._y[batch_idx])) + # _y.append(Counter(y[batch_idx]).most_common(1)[0][0]) + + _x_shuffled, _y_shuffled = shuffle_two_list_together(_x, _y) + if validation_split is not None: + _x_train, _y_train, _x_val, _y_val = train_val_split( + np.array(_x_shuffled), np.array(_y_shuffled) + ) + else: + _x_train, _y_train, _x_val, _y_val = ( + np.array(_x_shuffled), + np.array(_y_shuffled), + None, + None, + ) + return ( + _x_train, + one_hot(_y_train), + ), None if _x_val is None or _y_val is None else (_x_val, one_hot(_y_val)) + + @classmethod + def data_with_normalized_key_points(cls, pth): + pass + + @classmethod + def data_with_normalized_distance_matrix(cls, pth): + df = load_pickle(pth) + subset = df[ + [ + "normalized_distance_matrix", + "class_label", + "class_label_index", + "frame_details", + "frame_number", + ] + ] + x = np.array(subset["normalized_distance_matrix"].tolist()) + y = np.array(subset["class_label_index"].tolist()) + + return cls(x, y) + + @classmethod + def data_with_distance_matrix(cls, pth): + pass + + @classmethod + def data_with_key_points(cls, pth): + pass diff --git a/ivu/inference.py b/ivu/inference.py new file mode 100644 index 0000000..6d70d64 --- /dev/null +++ b/ivu/inference.py @@ -0,0 +1,45 @@ +from collections import defaultdict + +import numpy as np +import tensorflow +from ivu.conf import DataConf +from ivu.data.infer import VideoInferenceInputData + + +class Inference: + def __init__(self, model: tensorflow.keras.Model, conf: DataConf): + self._model = model + self._conf = conf + + @staticmethod + def _load_model(pth): + model = tensorflow.keras.models.load_model(pth, compile=True) + return model + + def run(self): + + pose_estimator_param = self._conf.get_pose_estimators_parameters() + video_param = self._conf.get_video_parameters() + inference_param = self._conf.get_inference_parameters() + + video_inference = VideoInferenceInputData( + data_dir=self._conf.get_entry("data_dir"), + **{**pose_estimator_param, **video_param, **inference_param} + ) + + predictions = defaultdict(list) + for ( + file_iterator, + file, + frame, + input_data, + ) in video_inference.data_for_normalized_distance_matrix(): + input_data = np.expand_dims(input_data, axis=0) + prediction = self._model.predict(input_data) + predictions[file].append(np.argmax(prediction)) + print(np.argmax(prediction)) + + @classmethod + def init_inference_from_config(cls, pth): + conf = DataConf(pth) + return cls(cls._load_model(conf.get_saved_model_pth()), conf) diff --git a/ivu/trainer.py b/ivu/trainer.py index 643f0b8..dbf5aea 100644 --- a/ivu/trainer.py +++ b/ivu/trainer.py @@ -1,5 +1,5 @@ -from ivu.conf import Config -from ivu.dataset import TrainInputData +from ivu.conf import TrainConf +from ivu.data.training import TrainInputData class Trainer: @@ -10,7 +10,11 @@ def __init__(self, train_data, val_data, config): def start_training(self): model = self._config.get_model() - model.compile(optimizer=self._config.get_optimizer(), loss=self._config.get_loss(), metrics="accuracy") + model.compile( + optimizer=self._config.get_optimizer(), + loss=self._config.get_loss(), + metrics="accuracy", + ) _ = model.fit( self._train_data[0], @@ -32,7 +36,7 @@ def train_with_normalized_key_points(cls): @classmethod def train_with_normalized_distance_matrix(cls, data_pth, conf_pth): input_data = TrainInputData.data_with_normalized_distance_matrix(data_pth) - conf = Config(conf_pth) + conf = TrainConf(conf_pth) parameters = conf.get_entry("data") train_data, val_data = input_data.create_sequence_data( diff --git a/ivu/utils.py b/ivu/utils.py index 54dc902..6a62d8d 100644 --- a/ivu/utils.py +++ b/ivu/utils.py @@ -1,6 +1,7 @@ import pickle import os import random +import tempfile from collections import defaultdict from typing import List @@ -8,6 +9,39 @@ import pandas as pd import numpy as np from decord import VideoReader, cpu +from scipy.spatial.distance import squareform, pdist + +from ivu.pose_estimator.pose_landmarks import ( + landmarks_to_embedding, + Pose16LandmarksBodyModel, +) + + +def log_in_tmp_dir(): + # https://stackoverflow.com/questions/847850/cross-platform-way-of-getting-temp-directory-in-python + log_dir = os.path.join(tempfile.gettempdir(), "ivu_logs") + if not os.path.exists: + os.makedirs(log_dir) + return log_dir + + +def log_in_current_dir(): + log_dir = os.path.join(os.getcwd(), "ivu_logs") + if not os.path.exists: + os.makedirs(log_dir) + return log_dir + + +def get_class_association(df): + class_ass = dict() + for i in range(0, 7): + d = filter_data_frame(df, "class_label_index", i)["class_label"] + class_ass[i] = np.unique(d.to_numpy())[0] + return class_ass + + +def filter_data_frame(df: pd.DataFrame, name, value): + return df.loc[df[name] == value] def shuffle_two_list_together(x, y): @@ -90,3 +124,56 @@ def read_video(pth, width=-1, height=-1) -> decord.VideoReader: def extract_from_data_frames(data_frame: pd.DataFrame, columns: List): return data_frame[columns] + + +def get_pose_data_from_rgb_frame(frame, pose_estimator): + body_key_points = pose_estimator.get_key_points_from_image(frame) + distance_matrix = squareform(pdist(np.array(body_key_points))) + + normalized_body_key_points = landmarks_to_embedding( + body_key_points, Pose16LandmarksBodyModel + ) + normalized_distance_matrix = squareform( + pdist(np.array(normalized_body_key_points[0])) + ) + + return ( + body_key_points, + distance_matrix, + normalized_body_key_points, + normalized_distance_matrix, + ) + + +def normalize_body_key_points(body_key_points, body_model): + return landmarks_to_embedding(body_key_points, body_model)[0] + + +def get_body_key_points(pose_estimator, rgb_input): + return pose_estimator.get_key_points_from_image(rgb_input) + + +def get_distance_matrix(pose_estimator, rgb_input): + body_key_points = pose_estimator.get_key_points_from_image(rgb_input) + return get_distance_matrix_from_key_points(np.array(body_key_points)) + + +def get_distance_matrix_from_key_points(body_key_points): + return squareform(pdist(np.array(body_key_points))) + + +def get_body_normalized_key_points(pose_estimator, rgb_input): + return normalize_body_key_points( + pose_estimator.get_key_points_from_image(rgb_input), Pose16LandmarksBodyModel + ) + + +def get_normalized_distance_matrix(pose_estimator, rgb_input): + body_key_points = get_body_normalized_key_points(pose_estimator, rgb_input) + return get_distance_matrix_from_key_points(body_key_points) + + +def get_normalized_distance_matrix_from_body_key_points(body_key_points): + return squareform( + pdist(normalize_body_key_points(body_key_points, Pose16LandmarksBodyModel)) + )