From 5650968ed59ed6476e36c48535edd0eef42625ce Mon Sep 17 00:00:00 2001 From: rauf Date: Wed, 18 Mar 2020 14:49:50 +0300 Subject: [PATCH] add deepfake --- configs/bengali.yml | 24 ++--- configs/deepfake.yml | 62 ++++++++++++ configs/road_signs.yml | 59 ++++++++++++ embedding_net/augmentations.py | 4 + embedding_net/backbones.py | 88 ++++++++++++++++- embedding_net/datagenerators.py | 55 ++++++----- embedding_net/model_new.py | 162 ++++---------------------------- embedding_net/utils.py | 25 ++++- test_network.ipynb | 151 ++++++++++++++++++++++++++--- train.py | 102 +++++++++++++++----- 10 files changed, 513 insertions(+), 219 deletions(-) create mode 100644 configs/deepfake.yml create mode 100644 configs/road_signs.yml diff --git a/configs/bengali.yml b/configs/bengali.yml index 1693606..8f8e408 100644 --- a/configs/bengali.yml +++ b/configs/bengali.yml @@ -3,8 +3,8 @@ MODEL: encodings_len: 256 mode : 'triplet' distance_type : 'l1' - backbone_name : 'efficientnet-b0' - backbone_weights : 'imagenet' + backbone_name : 'efficientnet-b5' + backbone_weights : 'noisy-student' freeze_backbone : False embeddings_normalization: True @@ -19,16 +19,16 @@ DATALOADER: GENERATOR: negatives_selection_mode : 'semihard' k_classes: 3 - k_samples: 5 - margin: 0.5 - batch_size : 10 - n_batches : 10 - augmentations : 'default' + k_samples: 3 + margin: 0.3 + batch_size : 8 + n_batches : 500 + augmentations : 'none' TRAIN: # optimizer parameters optimizer : 'radam' - learning_rate : 0.0001 + learning_rate : 0.00001 decay_factor : 0.99 step_size : 1 @@ -45,13 +45,13 @@ TRAIN: # decay_factor : 0.99 # step_size : 1 -# batch_size : 8 +# batch_size : 16 # val_steps : 200 -# steps_per_epoch : 10 -# n_epochs : 1 +# steps_per_epoch : 1000 +# n_epochs : 50 SAVE_PATHS: - project_name : 'bengali_efficientnet' + project_name : 'bengali_efn_b5' work_dir : 'work_dirs/' ENCODINGS: diff --git a/configs/deepfake.yml b/configs/deepfake.yml new file mode 100644 index 0000000..7327c3b --- /dev/null +++ b/configs/deepfake.yml @@ -0,0 +1,62 @@ +MODEL: + input_shape : [224, 224, 3] + encodings_len: 256 + mode : 'triplet' + distance_type : 'l1' + backbone_name : 'efficientnet-b3' + backbone_weights : 'noisy-student' + freeze_backbone : False + embeddings_normalization: True + +DATALOADER: + dataset_path : '/home/rauf/datasets/aaaa/deepfake/' + csv_file : + image_id_column : + label_column : + validate : True + val_ratio : 0.2 + +GENERATOR: + negatives_selection_mode : 'hardest' + k_classes: 2 + k_samples: 3 + margin: 0.5 + batch_size : 8 + n_batches : 7000 + augmentations : 'deepfake' + +TRAIN: + # optimizer parameters + optimizer : 'radam' + learning_rate : 0.000016 + decay_factor : 0.99 + step_size : 1 + + # embeddings learning training parameters + n_epochs : 1000 + + # plot training history + plot_history : True + +# SOFTMAX_PRETRAINING: +# # softmax pretraining parameters +# optimizer : 'radam' +# learning_rate : 0.0001 +# decay_factor : 0.99 +# step_size : 1 + +# batch_size : 16 +# val_steps : 200 +# steps_per_epoch : 1000 +# n_epochs : 50 + +SAVE_PATHS: + project_name : 'deepfake_efn_b3' + work_dir : 'work_dirs/' + +ENCODINGS: + # encodings parameters + save_encodings : True + centers_only: False + max_num_samples_of_each_class : 30 + knn_k : 1 \ No newline at end of file diff --git a/configs/road_signs.yml b/configs/road_signs.yml new file mode 100644 index 0000000..f5a3e79 --- /dev/null +++ b/configs/road_signs.yml @@ -0,0 +1,59 @@ +MODEL: + input_shape : [48, 48, 3] + encodings_len: 256 + mode : 'triplet' + distance_type : 'l1' + backbone_name : 'efficientnet-b0' + backbone_weights : 'imagenet' + freeze_backbone : False + embeddings_normalization: True + +DATALOADER: + dataset_path : '/home/rauf/datasets/road_signs/road_signs_separated/train/' + validate : True + val_ratio : 0.2 + +GENERATOR: + negatives_selection_mode : 'semihard' + k_classes: 2 + k_samples: 3 + margin: 0.5 + batch_size : 1 + n_batches : 1 + augmentations : 'none' + +TRAIN: + # optimizer parameters + optimizer : 'adam' + learning_rate : 0.0001 + decay_factor : 0.99 + step_size : 1 + + # embeddings learning training parameters + n_epochs : 1000 + + # plot training history + plot_history : True + +SOFTMAX_PRETRAINING: + # softmax pretraining parameters + optimizer : 'radam' + learning_rate : 0.0001 + decay_factor : 0.99 + step_size : 1 + + batch_size : 8 + val_steps : 200 + steps_per_epoch : 10 + n_epochs : 1 + +SAVE_PATHS: + project_name : 'road_signs_efficientnet' + work_dir : 'work_dirs/' + +ENCODINGS: + # encodings parameters + save_encodings : True + centers_only: False + max_num_samples_of_each_class : 30 + knn_k : 1 \ No newline at end of file diff --git a/embedding_net/augmentations.py b/embedding_net/augmentations.py index 69e941f..3bb233f 100644 --- a/embedding_net/augmentations.py +++ b/embedding_net/augmentations.py @@ -27,6 +27,10 @@ def get_aug(name='default', input_shape=[48, 48, 3]): A.GaussNoise(var_limit=(50, 80), p=0.3), A.RandomCrop(p=0.8, height=2*input_shape[1]/3, width=2*input_shape[0]/3) ], p=1) + elif name == 'deepfake': + augmentations = A.Compose([ + A.HorizontalFlip(p=0.5), + ], p=1) elif name == 'plates2': augmentations = A.Compose([ A.CLAHE(clip_limit=(1,4),p=0.3), diff --git a/embedding_net/backbones.py b/embedding_net/backbones.py index 03361bc..551848a 100644 --- a/embedding_net/backbones.py +++ b/embedding_net/backbones.py @@ -3,7 +3,11 @@ from tensorflow.keras.models import Model from tensorflow.keras.regularizers import l2 import tensorflow.keras.backend as K - +from tensorflow.keras.callbacks import TensorBoard, LearningRateScheduler +from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint +from .datagenerators import SimpleDataGenerator +import os +import numpy as np def get_backbone(input_shape, encodings_len=4096, @@ -117,4 +121,86 @@ def get_backbone(input_shape, base_model = Model( inputs=[backbone_model.input], outputs=[encoded_output]) + base_model._make_predict_function() + return base_model, backbone_model + + +def pretrain_backbone_softmax(backbone_model, data_loader, params_softmax, params_save_paths): + + optimizer = params_softmax['optimizer'] + learning_rate = params_softmax['learning_rate'] + decay_factor = params_softmax['decay_factor'] + step_size = params_softmax['step_size'] + + input_shape = params_softmax['input_shape'] + batch_size = params_softmax['batch_size'] + val_steps = params_softmax['val_steps'] + steps_per_epoch = params_softmax['steps_per_epoch'] + n_epochs = params_softmax['n_epochs'] + augmentations = params_softmax['augmentations'] + + n_classes = data_loader.n_classes + + x = GlobalAveragePooling2D()(backbone_model.output) + + output = Dense(n_classes, activation='softmax')(x) + model = Model(inputs=[backbone_model.input], outputs=[output]) + + # train + model.compile(optimizer=optimizer, + loss='categorical_crossentropy', + metrics=['accuracy']) + + train_generator = SimpleDataGenerator(data_loader.train_data, + data_loader.class_names, + input_shape=input_shape, + batch_size = batch_size, + n_batches = steps_per_epoch, + augmentations=augmentations) + + if data_loader.validate: + val_generator = SimpleDataGenerator(data_loader.val_data, + data_loader.class_names, + input_shape=input_shape, + batch_size = batch_size, + n_batches = steps_per_epoch, + augmentations=augmentations) + checkpoint_callback_monitor = 'val_loss' + else: + val_generator = None + checkpoint_callback_monitor = 'loss' + + tensorboard_save_path = os.path.join( + params_save_paths['work_dir'], + params_save_paths['project_name'], + 'pretraining_model/tf_log/') + weights_save_file = os.path.join( + params_save_paths['work_dir'], + params_save_paths['project_name'], + 'pretraining_model/weights/', + params_save_paths['project_name']+'_{epoch:03d}_{val_acc:03f}' +'.h5') + + callbacks = [ + LearningRateScheduler(lambda x: learning_rate * + decay_factor ** np.floor(x/step_size)), + ReduceLROnPlateau(monitor=checkpoint_callback_monitor, factor=0.1, + patience=20, verbose=1), + EarlyStopping(monitor=checkpoint_callback_monitor, + patience=10, + verbose=1, + restore_best_weights=True), + # TensorBoard(log_dir=tensorboard_save_path), + ModelCheckpoint(filepath=weights_save_file, + verbose=1, + monitor=checkpoint_callback_monitor, + save_best_only=True)] + # checkpoints_load_name = 'work_dirs/bengali_efficientnet/pretraining_model/weights/bengali_efficientnet_020_0.932969.h5' + # model.load_weights(checkpoints_load_name, by_name=True) + history = model.fit_generator(train_generator, + steps_per_epoch=steps_per_epoch, + epochs=n_epochs, + verbose=1, + validation_data=val_generator, + validation_steps=val_steps, + callbacks=callbacks) \ No newline at end of file diff --git a/embedding_net/datagenerators.py b/embedding_net/datagenerators.py index 9795500..c9cb3cd 100644 --- a/embedding_net/datagenerators.py +++ b/embedding_net/datagenerators.py @@ -7,6 +7,9 @@ from sklearn.metrics import pairwise_distances from tensorflow.keras.utils import Sequence from sklearn.model_selection import train_test_split +from .utils import get_image +from tensorflow.keras import backend as K +import tensorflow as tf class ENDataLoader(): def __init__(self, dataset_path, @@ -40,6 +43,8 @@ def __init__(self, dataset_path, def split_train_val(self, val_ratio): train_data = {} val_data = {} + print(len(self.class_files_paths['real'])) + print(len(self.class_files_paths['fake'])) for k, v in self.class_files_paths.items(): train_d, val_d = train_test_split(v, test_size=val_ratio, random_state=42) train_data[k] = train_d @@ -57,13 +62,26 @@ def _load_from_dataframe(self, csv_file, image_id_column, label_column): def _load_from_directory(self): self.class_names = [f.name for f in os.scandir(self.dataset_path) if f.is_dir()] class_dir_paths = [f.path for f in os.scandir(self.dataset_path) if f.is_dir()] - + for class_name, class_dir_path in zip(self.class_names, class_dir_paths): - class_image_paths = [f.path for f in os.scandir(class_dir_path) if f.is_file() and - (f.name.endswith('.jpg') or - f.name.endswith('.png') and - not f.name.startswith('._'))] - self.class_files_paths[class_name] = class_image_paths + subdirs = [f.path for f in os.scandir(class_dir_path) if f.is_dir()] + self.class_files_paths[class_name] = [] + print(class_dir_path) + if len(subdirs)>0: + for subdir in subdirs: + class_image_paths = [f.path for f in os.scandir(subdir) if f.is_file() and + (f.name.endswith('.jpg') or + f.name.endswith('.png') and + not f.name.startswith('._'))] + for class_image_path in class_image_paths: + self.class_files_paths[class_name].append(class_image_path) + else: + class_image_paths = [f.path for f in os.scandir(class_dir_path) if f.is_file() and + (f.name.endswith('.jpg') or + f.name.endswith('.png') and + not f.name.startswith('._'))] + for class_image_path in class_image_paths: + self.class_files_paths[class_name].append(class_image_path) class ENDataGenerator(Sequence): @@ -89,16 +107,6 @@ def __len__(self): def __getitem__(self, index): pass - - def get_image(self, img_path): - img = cv2.imread(img_path) - if img is None: - print('image is not exist ' + img_path) - return None - if self.input_shape: - img = cv2.resize( - img, (self.input_shape[0], self.input_shape[1])) - return img def _get_images_set(self, clsss, idxs, with_aug=True): if type(clsss) is list: @@ -106,11 +114,7 @@ def _get_images_set(self, clsss, idxs, with_aug=True): else: img_paths = [self.class_files_paths[clsss][idx] for idx in idxs] - imgs = [cv2.imread(img_path) for img_path in img_paths] - - if self.input_shape: - imgs = [cv2.resize( - img, (self.input_shape[0], self.input_shape[1])) for img in imgs] + imgs = [get_image(img_path, self.input_shape) for img_path in img_paths] if with_aug: imgs = [self.augmentations(image=img)['image'] for img in imgs] @@ -141,10 +145,13 @@ def __init__(self, embedding_model, 'hardest': self.hardest_negative, 'random_hard': self.random_hard_negative} self.embedding_model = embedding_model + self.k_classes = k_classes self.k_samples = k_samples self.margin = margin self.negative_selection_fn = modes[negatives_selection_mode] + self.session = K.get_session() + self.graph = tf.get_default_graph() def hardest_negative(self, loss_values, margin=0.5): hard_negative = np.argmax(loss_values) @@ -169,11 +176,15 @@ def get_batch_triplets_mining(self): all_embeddings_list = [] all_images_list = [] + for idx, cl_img_idxs in enumerate(selected_images): images = self._get_images_set(selected_classes[idx], cl_img_idxs, with_aug=self.augmentations) all_images_list.append(images) - embeddings = self.embedding_model.predict(images) + with self.session.as_default(): + with self.graph.as_default(): + embeddings = self.embedding_model.predict(images) all_embeddings_list.append(embeddings) + all_embeddings = np.vstack(all_embeddings_list) all_images = np.vstack(all_images_list) distance_matrix = pairwise_distances(all_embeddings) diff --git a/embedding_net/model_new.py b/embedding_net/model_new.py index 65276ba..bc8ee57 100644 --- a/embedding_net/model_new.py +++ b/embedding_net/model_new.py @@ -8,13 +8,11 @@ from tensorflow.keras.layers import Dense, Input, Lambda, concatenate, GlobalAveragePooling2D import pickle from .utils import load_encodings, parse_params -from .datagenerators import ENDataLoader, SimpleDataGenerator, TripletsDataGenerator, SimpleTripletsDataGenerator, SiameseDataGenerator from .backbones import get_backbone from . import losses_and_accuracies as lac +from .utils import get_image, get_images import matplotlib.pyplot as plt from sklearn.neighbors import KNeighborsClassifier -from tensorflow.keras.callbacks import TensorBoard, LearningRateScheduler -from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint class EmbeddingNet: @@ -31,140 +29,42 @@ def __init__(self, params): self.backbone_model = None self.encoded_training_data = {} - self.dataloader = None - self.train_generator = None - self.val_generator = None - - def pretrain_backbone_softmax(self): - - optimizer = self.params_softmax['optimizer'] - learning_rate = self.params_softmax['learning_rate'] - decay_factor = self.params_softmax['decay_factor'] - step_size = self.params_softmax['step_size'] - - input_shape = self.params_softmax['input_shape'] - batch_size = self.params_softmax['batch_size'] - val_steps = self.params_softmax['val_steps'] - steps_per_epoch = self.params_softmax['steps_per_epoch'] - n_epochs = self.params_softmax['n_epochs'] - augmentations = self.params_softmax['augmentations'] - - n_classes = self.data_loader.n_classes - - x = GlobalAveragePooling2D()(self.backbone_model.output) - - output = Dense(n_classes, activation='softmax')(x) - model = Model(inputs=[self.backbone_model.input], outputs=[output]) - - # train - model.compile(optimizer=optimizer, - loss='categorical_crossentropy', - metrics=['accuracy']) - - train_generator = SimpleDataGenerator(self.data_loader.train_data, - self.data_loader.class_names, - input_shape=input_shape, - batch_size = batch_size, - n_batches = steps_per_epoch, - augmentations=augmentations) - - if self.data_loader.validate: - val_generator = SimpleDataGenerator(self.data_loader.val_data, - self.data_loader.class_names, - input_shape=input_shape, - batch_size = batch_size, - n_batches = steps_per_epoch, - augmentations=augmentations) - checkpoint_callback_monitor = 'val_loss' - else: - val_generator = None - checkpoint_callback_monitor = 'loss' - - tensorboard_save_path = os.path.join( - self.params_save_paths['work_dir'], - self.params_save_paths['project_name'], - 'pretraining_model/tf_log/') - weights_save_file = os.path.join( - self.params_save_paths['work_dir'], - self.params_save_paths['project_name'], - 'pretraining_model/weights/', - self.params_save_paths['project_name']+'.h5') - - callbacks = [ - LearningRateScheduler(lambda x: learning_rate * - decay_factor ** np.floor(x/step_size)), - ReduceLROnPlateau(monitor=checkpoint_callback_monitor, factor=0.1, - patience=20, verbose=1), - EarlyStopping(monitor=checkpoint_callback_monitor, - patience=10, - verbose=1, - restore_best_weights=True), - TensorBoard(log_dir=tensorboard_save_path), - ModelCheckpoint(filepath=weights_save_file, - verbose=1, - monitor=checkpoint_callback_monitor, - save_best_only=True)] - - history = model.fit_generator(train_generator, - steps_per_epoch=steps_per_epoch, - epochs=n_epochs, - verbose=1, - validation_data=val_generator, - validation_steps=val_steps, - callbacks=callbacks) def _create_base_model(self): self.base_model, self.backbone_model = get_backbone(**self.params_model) - - def _create_dataloader(self): - self.data_loader = ENDataLoader(**self.params_dataloader) - - def _create_generators(self): - pass - def _generate_encoding(self, img_path): - img = self.data_loader.get_image(img_path) - if img is None: - return None - encoding = self.base_model.predict(np.expand_dims(img, axis=0)) - return encoding + def _generate_encodings(self, imgs): + encodings = self.base_model.predict(imgs) + return encodings - def train(self, callbacks=[], verbose=1): - history = self.model.fit_generator(self.train_generator, - validation_data=self.val_generator, - epochs=self.params_train['n_epochs'], - callbacks=callbacks, - verbose=verbose, - use_multiprocessing=True) - - return history - def train_embeddings_classifier(self, + def train_embeddings_classifier(self, data_loader, classification_model, max_n_samples=10, shuffle=True): - encodings = self.generate_encodings(max_n_samples=max_n_samples, + encodings = self.generate_encodings(data_loader, max_n_samples=max_n_samples, shuffle=shuffle) classification_model.fit(encodings['encodings'], encodings['labels']) - def generate_encodings(self, max_n_samples=10, + def generate_encodings(self, data_loader, max_n_samples=10, shuffle=True): data_paths, data_labels, data_encodings = [], [], [] encoded_training_data = {} - for class_name in self.data_loader.class_names: - data_list = self.data_loader.train_data[class_name] + for class_name in data_loader.class_names: + data_list = data_loader.train_data[class_name] if len(data_list)>max_n_samples: if shuffle: random.shuffle(data_list) data_list = data_list[:max_n_samples] data_paths += data_list - data_labels += class_name * len(data_list) - for img_path in data_list: - encod = self._generate_encoding(img_path) + imgs = get_images(data_list) + encods = self._generate_encodings(imgs) + for encod in encods: data_encodings.append(encod) + data_labels.append(class_name) encoded_training_data['paths'] = data_paths encoded_training_data['labels'] = data_labels @@ -182,7 +82,7 @@ def load_model(self, file_path): self.model = load_model(file_path, custom_objects={'contrastive_loss': lac.contrastive_loss, 'accuracy': lac.accuracy, - 'loss_function': lac.triplet_loss(self.margin), + 'loss_function': lac.triplet_loss(self.params_generator['margin']), 'RAdam': RAdam}) self.input_shape = list(self.model.inputs[0].shape[1:]) self.base_model = Model(inputs=[self.model.layers[3].get_input_at(0)], @@ -218,15 +118,15 @@ def predict_knn(self, image, with_top5=False): else: return predicted_label - def calculate_prediction_accuracy(self): + def calculate_prediction_accuracy(self, data_loader): correct_top1 = 0 correct_top5 = 0 accuracies = {'top1':0, 'top5':0 } - total_n_of_images = len(self.data_loader.images_paths['val']) - for img_path, img_label in zip(self.data_loader.images_paths['val'], - self.data_loader.images_labels['val']): + total_n_of_images = len(data_loader.images_paths['val']) + for img_path, img_label in zip(data_loader.images_paths['val'], + data_loader.images_labels['val']): prediction, prediction_top5 = self.predict_knn(img_path, with_top5=True) if prediction[0] == img_label: correct_top1 += 1 @@ -243,24 +143,12 @@ class TripletNet(EmbeddingNet): def __init__(self, params, training=False): super().__init__(params) self._create_base_model() - self.base_model._make_predict_function() self.training = training if self.training: - self._create_dataloader() - self._create_generators() self._create_model_triplet() - def _create_generators(self): - self.train_generator = TripletsDataGenerator(embedding_model=self.base_model, - class_files_paths=self.data_loader.train_data, - class_names=self.data_loader.class_names, - **self.params_generator) - if self.data_loader.validate: - self.val_generator = SimpleTripletsDataGenerator(self.data_loader.val_data, - self.data_loader.class_names, - **self.params_generator) def _create_model_triplet(self): input_image_a = Input(self.params_model['input_shape']) @@ -291,20 +179,6 @@ class SiameseNet(EmbeddingNet): def __init__(self, params, training): super().__init__(params) self._create_model_siamese() - if training: - self.dataloader = {} - self.train_generator = {} - self.val_generator = {} - self._create_generators() - - def _create_generators(self): - self.train_generator = SiameseDataGenerator(class_files_paths=self.data_loader.train_data, - class_names=self.data_loader.class_names, - **self.params_generator) - if self.data_loader.validate: - self.val_generator = SiameseDataGenerator(class_files_paths=self.data_loader.val_data, - class_names=self.data_loader.class_names, - **self.params_generator) def _create_model_siamese(self): diff --git a/embedding_net/utils.py b/embedding_net/utils.py index 1f19e7d..30acd11 100644 --- a/embedding_net/utils.py +++ b/embedding_net/utils.py @@ -10,6 +10,22 @@ from .augmentations import get_aug +def get_image(img_path, input_shape=None): + img = cv2.imread(img_path) + if img is None: + print('image is not exist ' + img_path) + return None + if input_shape: + img = cv2.resize( + img, (input_shape[0], input_shape[1])) + return img + +def get_images(img_paths, input_shape=None): + imgs = [get_image(img_path, input_shape) for img_path in img_paths] + return np.array(imgs) + + + def load_encodings(path_to_encodings): with open(path_to_encodings, 'rb') as f: @@ -42,9 +58,10 @@ def plot_tsne(encodings_path, save_plot_dir, show=True): fig.savefig("{}{}.png".format(save_plot_dir, 'tsne.png')) -def plot_tsne_interactive(encodings_path): +def plot_tsne_interactive(encodings): import plotly.graph_objects as go - encodings = load_encodings(encodings_path) + if type(encodings) is str: + encodings = load_encodings(encodings) labels = list(set(encodings['labels'])) tsne = TSNE() tsne_train = tsne.fit_transform(encodings['encodings']) @@ -60,8 +77,8 @@ def plot_tsne_interactive(encodings_path): mode='markers', marker=dict(color=color, size=10), - text=l, - name=l)) + text=str(l), + name=str(l))) fig.update_layout( title=go.layout.Title(text="t-SNE plot", xref="paper", diff --git a/test_network.ipynb b/test_network.ipynb index d076447..be723f8 100644 --- a/test_network.ipynb +++ b/test_network.ipynb @@ -181,7 +181,9 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "heading_collapsed": true + }, "source": [ "#### Test triplets dataloader with negative mining techniques" ] @@ -189,7 +191,9 @@ { "cell_type": "code", "execution_count": 9, - "metadata": {}, + "metadata": { + "hidden": true + }, "outputs": [ { "name": "stdout", @@ -651,7 +655,9 @@ { "cell_type": "code", "execution_count": 10, - "metadata": {}, + "metadata": { + "hidden": true + }, "outputs": [ { "ename": "ValueError", @@ -674,7 +680,9 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "heading_collapsed": true + }, "source": [ "#### Test new loader" ] @@ -682,7 +690,9 @@ { "cell_type": "code", "execution_count": 2, - "metadata": {}, + "metadata": { + "hidden": true + }, "outputs": [ { "data": { @@ -736,19 +746,134 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "WARNING:tensorflow:From /home/rauf/anaconda3/envs/bengali_embeddings/lib/python3.7/site-packages/tensorflow_core/python/util/deprecation.py:507: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with distribution=normal is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "`normal` is a deprecated alias for `truncated_normal`\n", + "WARNING:tensorflow:From /home/rauf/anaconda3/envs/bengali_embeddings/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "If using Keras pass *_constraint arguments to layers.\n", + "WARNING:tensorflow:From /home/rauf/anaconda3/envs/bengali_embeddings/lib/python3.7/site-packages/tensorflow_core/python/ops/init_ops.py:97: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Call initializer instance with the dtype argument instead of passing it to the constructor\n", + "WARNING:tensorflow:From /home/rauf/anaconda3/envs/bengali_embeddings/lib/python3.7/site-packages/tensorflow_core/python/ops/init_ops.py:97: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Call initializer instance with the dtype argument instead of passing it to the constructor\n", + "WARNING:tensorflow:From /home/rauf/anaconda3/envs/bengali_embeddings/lib/python3.7/site-packages/tensorflow_core/python/ops/init_ops.py:97: calling Ones.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Call initializer instance with the dtype argument instead of passing it to the constructor\n", + "WARNING:tensorflow:From /home/rauf/anaconda3/envs/bengali_embeddings/lib/python3.7/site-packages/tensorflow_core/python/ops/init_ops.py:97: calling GlorotUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Call initializer instance with the dtype argument instead of passing it to the constructor\n", + "WARNING:tensorflow:From /home/rauf/anaconda3/envs/bengali_embeddings/lib/python3.7/site-packages/tensorflow_core/python/ops/math_grad.py:1424: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Use tf.where in 2.0, which has the same broadcast rule as np.where\n" + ] + } + ], "source": [ - "config_name = 'plates_resnext50'\n", - "cfg_params = parse_net_params('configs/{}.yml'.format(config_name))\n", - "model = EmbeddingNet(cfg_params, training = False)\n", - "model.load_model('work_dirs/{}/weights/best_model_{}.h5'.format(config_name, config_name))\n", - "model.generate_encodings(save_file_name='work_dirs/{}/encodings/encodings_{}.pkl'.format(config_name, config_name),\n", - " max_num_samples_of_each_class=30, knn_k = 1, shuffle=True)\n", + "from embedding_net.model_new import TripletNet\n", + "from embedding_net.datagenerators import ENDataLoader\n", + "from embedding_net.utils import parse_params\n", + "from embedding_net.utils import plot_tsne_interactive\n", + "\n", + "config_name = 'bengali'\n", + "model_name = 'best_bengali_efficientnet_028_0.023263.h5'\n", + "cfg_params = parse_params('configs/{}.yml'.format(config_name))\n", + "model = TripletNet(cfg_params, training = False)\n", + "model.load_model('work_dirs/{}/weights/{}'.format(config_name, model_name))\n", + "data_loader = ENDataLoader(**cfg_params['dataloader'])\n", + "\n", + "# encoded_training_data = model.generate_encodings(data_loader,\n", + "# max_n_samples=30, shuffle=True)\n", + "\n", + "encoded_training_data = model.generate_encodings(data_loader,\n", + " max_n_samples=50000000000000000, shuffle=False)\n", + "model.save_encodings(encoded_training_data)\n", + "# plot_tsne_interactive(encoded_training_data)\n", "# model.load_encodings('encodings/road_signs/encodings_{}.pkl')" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Train sklearn_model" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['classif_model.sav']" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from sklearn.linear_model import LogisticRegression\n", + "from sklearn.pipeline import Pipeline\n", + "from sklearn.metrics import accuracy_score\n", + "from sklearn.multiclass import OneVsRestClassifier\n", + "from sklearn.neighbors import KNeighborsClassifier\n", + "import pickle\n", + "import joblib\n", + "\n", + "k_val = 3\n", + "classification_model = KNeighborsClassifier(n_neighbors=k_val)\n", + "with open('encodings.pkl', 'rb') as f:\n", + " data = pickle.load(f)\n", + "classification_model.fit(data['encodings'], data['labels'])\n", + "joblib.dump(classification_model,'classif_model.sav')" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2]\n", + "[93]\n", + "[19]\n", + "[115]\n", + "[55]\n", + "[115]\n", + "[147]\n", + "[137]\n", + "[119]\n", + "[133]\n", + "[154]\n", + "[21]\n" + ] + } + ], + "source": [ + "import cv2\n", + "dataset_path = '/home/rauf/datasets/bengali/pngs/test/'\n", + "for i in range(12):\n", + " img = cv2.imread(dataset_path+'Test_{}.png'.format(i))\n", + " encoding = model.base_model.predict(np.expand_dims(img, axis=0))\n", + " predicted_label = classification_model.predict(encoding)\n", + " print(predicted_label)" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/train.py b/train.py index 28c7bf2..3273223 100644 --- a/train.py +++ b/train.py @@ -3,8 +3,12 @@ from embedding_net.model_new import EmbeddingNet, TripletNet from tensorflow.keras.callbacks import TensorBoard, LearningRateScheduler from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint +from embedding_net.datagenerators import ENDataLoader, SimpleDataGenerator, TripletsDataGenerator, SimpleTripletsDataGenerator, SiameseDataGenerator from embedding_net.utils import parse_params, plot_grapths +from embedding_net.backbones import pretrain_backbone_softmax import argparse +from tensorflow import keras +import tensorflow as tf def parse_args(): @@ -25,7 +29,7 @@ def create_save_folders(params): plots_save_path = os.path.join(work_dir_path, 'plots/') tensorboard_save_path = os.path.join(work_dir_path, 'tf_log/') tensorboard_pretrained_save_path = os.path.join(work_dir_path, 'pretraining_model/tf_log/') - weights_save_file_path = os.path.join(weights_save_path, 'best_' + params['project_name'] + '.h5') + weights_save_file_path = os.path.join(weights_save_path, 'best_' + params['project_name']+'_{epoch:03d}_{loss:03f}' + '.h5') os.makedirs(work_dir_path , exist_ok=True) os.makedirs(weights_save_path, exist_ok=True) @@ -37,21 +41,33 @@ def create_save_folders(params): return tensorboard_save_path, weights_save_file_path, plots_save_path def main(): + config = tf.ConfigProto( + device_count={'GPU': 1}, + intra_op_parallelism_threads=1, + allow_soft_placement=True + ) + + config.gpu_options.allow_growth = True + # config.gpu_options.per_process_gpu_memory_fraction = 1 + + session = tf.Session(config=config) + + keras.backend.set_session(session) + args = parse_args() cfg_params = parse_params(args.config) params_train = cfg_params['train'] params_dataloader = cfg_params['dataloader'] + params_generator = cfg_params['generator'] tensorboard_save_path, weights_save_file_path, plots_save_path = create_save_folders(cfg_params['save_paths']) - model = TripletNet(cfg_params, training=True) - if 'softmax' in cfg_params: - model.pretrain_backbone_softmax() + cfg_params['save_paths'] + work_dir_path = os.path.join(cfg_params['save_paths']['work_dir'], + cfg_params['save_paths']['project_name']) + weights_save_path = os.path.join(work_dir_path, 'weights/') - if args.resume_from is not None: - model.load_model(args.resume_from) - initial_lr = params_train['learning_rate'] decay_factor = params_train['decay_factor'] @@ -70,28 +86,68 @@ def main(): EarlyStopping(monitor=callback_monitor, patience=10, verbose=1), - TensorBoard(log_dir=tensorboard_save_path), + # TensorBoard(log_dir=tensorboard_save_path), ModelCheckpoint(filepath=weights_save_file_path, - verbose=1, monitor=callback_monitor, save_best_only=True) + monitor=callback_monitor, + save_best_only=True, + verbose=1) ] - history = model.train(callbacks=callbacks) + data_loader = ENDataLoader(**params_dataloader) + model = TripletNet(cfg_params, training=True) + if args.resume_from is not None: + model.load_model(args.resume_from) + + model.load_model('work_dirs/deepfake_efn_b3/weights/best_deepfake_efn_b3_001_0.430115.h5') + + if 'softmax' in cfg_params: + params_softmax = cfg_params['softmax'] + params_save_paths = cfg_params['save_paths'] + pretrain_backbone_softmax(model.backbone_model, + data_loader, + params_softmax, + params_save_paths) + + # def _create_generators(self): + # self.train_generator = SiameseDataGenerator(class_files_paths=self.data_loader.train_data, + # class_names=self.data_loader.class_names, + # **self.params_generator) + # if self.data_loader.validate: + # self.val_generator = SiameseDataGenerator(class_files_paths=self.data_loader.val_data, + # class_names=self.data_loader.class_names, + # **self.params_generator) + + # train_generator = SimpleTripletsDataGenerator(class_files_paths=data_loader.train_data, + # class_names=data_loader.class_names, + # **params_generator) + # checkpoints_load_name = 'work_dirs/bengali_efn_b5/pretraining_model/weights/best_efficientnet-b5.hdf5' + # model.base_model.load_weights(checkpoints_load_name, by_name=True) + train_generator = TripletsDataGenerator(embedding_model=model.base_model, + class_files_paths=data_loader.train_data, + class_names=data_loader.class_names, + **params_generator) + - if params_train['plot_history']: - plot_grapths(history, plots_save_path) + if data_loader.validate: + val_generator = SimpleTripletsDataGenerator(data_loader.val_data, + data_loader.class_names, + **params_generator) + else: + val_generator = None + + history = model.model.fit_generator(train_generator, + validation_data=val_generator, + epochs=params_train['n_epochs'], + callbacks=callbacks, + verbose=1, + use_multiprocessing=False) - # if cfg_params['save_encodings']: - # encodings_save_file = os.path.join( - # encodings_save_path, cfg_params['encodings_save_name']) - # model.generate_encodings(save_file_name=encodings_save_file, - # max_num_samples_of_each_class=cfg_params['max_num_samples_of_each_class'], - # knn_k=cfg_params['knn_k'], - # shuffle=True) - # if cfg_params['to_validate']: - # model_accuracies = model.calculate_prediction_accuracy() - # print('Model top1 accuracy on validation set: {}'.format(model_accuracies['top1'])) - # print('Model top5 accuracy on validation set: {}'.format(model_accuracies['top5'])) + # encoded_training_data = model.generate_encodings(data_loader, + # max_n_samples=50000000000000000, shuffle=False) + # model.save_encodings(encoded_training_data) + if params_train['plot_history']: + plot_grapths(history, plots_save_path) if __name__ == '__main__': main()