From 11affca3785d34fabe28864ff38c8d6e4cda1286 Mon Sep 17 00:00:00 2001 From: rauf Date: Thu, 19 Mar 2020 19:17:39 +0300 Subject: [PATCH] add some changes --- configs/deepfake.yml | 2 +- configs/deepfake_siamese.yml | 59 +++++++++++++++++++++ embedding_net/datagenerators.py | 76 +++++++++++++++++---------- embedding_net/model_new.py | 70 ++++++++++++++----------- get_classification_model.py | 14 +++++ train.py | 91 +++++++++++++++++++++------------ 6 files changed, 221 insertions(+), 91 deletions(-) create mode 100644 configs/deepfake_siamese.yml create mode 100644 get_classification_model.py diff --git a/configs/deepfake.yml b/configs/deepfake.yml index 7327c3b..a491a1c 100644 --- a/configs/deepfake.yml +++ b/configs/deepfake.yml @@ -2,7 +2,7 @@ MODEL: input_shape : [224, 224, 3] encodings_len: 256 mode : 'triplet' - distance_type : 'l1' + distance_type : 'l2' backbone_name : 'efficientnet-b3' backbone_weights : 'noisy-student' freeze_backbone : False diff --git a/configs/deepfake_siamese.yml b/configs/deepfake_siamese.yml new file mode 100644 index 0000000..c1acc6c --- /dev/null +++ b/configs/deepfake_siamese.yml @@ -0,0 +1,59 @@ +MODEL: + input_shape : [224, 224, 3] + encodings_len: 256 + mode : 'siamese' + distance_type : 'l1' + backbone_name : 'efficientnet-b3' + backbone_weights : 'noisy-student' + freeze_backbone : False + embeddings_normalization: True + +DATALOADER: + dataset_path : '/home/rauf/datasets/aaaa/deepfake/' + csv_file : + image_id_column : + label_column : + validate : True + val_ratio : 0.2 + +GENERATOR: + batch_size : 5 + n_batches : 5000 + n_batches_val : 500 + augmentations : 'deepfake' + +TRAIN: + # optimizer parameters + optimizer : 'radam' + learning_rate : 0.00016 + decay_factor : 0.95 + step_size : 1 + + # embeddings learning training parameters + n_epochs : 1000 + + # plot training history + plot_history : True + +# SOFTMAX_PRETRAINING: +# # softmax pretraining parameters +# optimizer : 'radam' +# learning_rate : 0.0001 +# decay_factor : 0.99 +# step_size : 1 + +# batch_size : 16 +# val_steps : 200 +# steps_per_epoch : 1000 +# n_epochs : 50 + +SAVE_PATHS: + project_name : 'deepfake_efn_b3' + work_dir : 'work_dirs/' + +ENCODINGS: + # encodings parameters + save_encodings : True + centers_only: False + max_num_samples_of_each_class : 30 + knn_k : 1 \ No newline at end of file diff --git a/embedding_net/datagenerators.py b/embedding_net/datagenerators.py index c9cb3cd..0482171 100644 --- a/embedding_net/datagenerators.py +++ b/embedding_net/datagenerators.py @@ -57,15 +57,16 @@ def _load_from_dataframe(self, csv_file, image_id_column, label_column): for class_name in self.class_names: image_names = dataframe.loc[dataframe[label_column] == class_name][image_id_column] image_paths = [os.path.join(self.dataset_path, f) for f in image_names] - self.class_files_paths[class_name] = image_paths + self.class_files_paths[class_name] = image_paths def _load_from_directory(self): self.class_names = [f.name for f in os.scandir(self.dataset_path) if f.is_dir()] class_dir_paths = [f.path for f in os.scandir(self.dataset_path) if f.is_dir()] - + for class_name, class_dir_path in zip(self.class_names, class_dir_paths): subdirs = [f.path for f in os.scandir(class_dir_path) if f.is_dir()] - self.class_files_paths[class_name] = [] + # self.class_files_paths[class_name] = [] + temp_list = [] print(class_dir_path) if len(subdirs)>0: for subdir in subdirs: @@ -73,29 +74,32 @@ def _load_from_directory(self): (f.name.endswith('.jpg') or f.name.endswith('.png') and not f.name.startswith('._'))] - for class_image_path in class_image_paths: - self.class_files_paths[class_name].append(class_image_path) + temp_list.extend(class_image_paths) else: class_image_paths = [f.path for f in os.scandir(class_dir_path) if f.is_file() and (f.name.endswith('.jpg') or f.name.endswith('.png') and not f.name.startswith('._'))] - for class_image_path in class_image_paths: - self.class_files_paths[class_name].append(class_image_path) + temp_list.extend(class_image_paths) + self.class_files_paths[class_name] = temp_list class ENDataGenerator(Sequence): def __init__(self, class_files_paths, class_names, + val_gen = False, input_shape=None, batch_size = 32, n_batches = 10, + n_batches_val = 10, augmentations=None): self.input_shape = input_shape self.augmentations = augmentations self.batch_size = batch_size self.n_batches = n_batches + self.n_batches_val = n_batches_val + self.val_gen = val_gen self.class_files_paths = class_files_paths self.class_names = class_names @@ -103,7 +107,10 @@ def __init__(self, class_files_paths, self.n_samples = {k: len(v) for k, v in self.class_files_paths.items()} def __len__(self): - return self.n_batches + if self.val_gen: + return self.n_batches_val + else: + return self.n_batches def __getitem__(self, index): pass @@ -119,7 +126,7 @@ def _get_images_set(self, clsss, idxs, with_aug=True): if with_aug: imgs = [self.augmentations(image=img)['image'] for img in imgs] - return np.array(imgs) + return np.array(imgs)/255. class TripletsDataGenerator(ENDataGenerator): @@ -289,60 +296,75 @@ class SiameseDataGenerator(ENDataGenerator): def __init__(self, class_files_paths, class_names, + val_gen = False, input_shape=None, batch_size = 32, - n_batches = 10, + n_batches = 10, + n_batches_val = 10, augmentations=None): super().__init__(class_files_paths=class_files_paths, - class_names=class_names, + class_names=class_names, + val_gen = False, input_shape=input_shape, batch_size=batch_size, - n_batches=n_batches, + n_batches=n_batches, + n_batches_val = 10, augmentations=augmentations) def get_batch_pairs(self): pairs = [np.zeros((self.batch_size, self.input_shape[0], self.input_shape[1], 3)), np.zeros( (self.batch_size, self.input_shape[0], self.input_shape[1], 3))] - targets = np.zeros((self.batch_size,)) + targets = np.zeros((self.batch_size, )) + targets1 = np.zeros((self.batch_size, )) + targets2 = np.zeros((self.batch_size, )) + # targets = [] n_same_class = self.batch_size // 2 selected_class_idx = random.randrange(0, self.n_classes) selected_class = self.class_names[selected_class_idx] - selected_class_n_elements = len(self.indexes[selected_class]) + selected_class_n_elements = self.n_samples[selected_class] - indxs = np.random.randint( - selected_class_n_elements, size=self.batch_size) + if selected_class == 'real': + t_id = 0 + f_id = 1 + else: + t_id = 1 + f_id = 0 + indxs = np.random.randint(selected_class_n_elements, size=self.batch_size) with_aug = self.augmentations count = 0 for i in range(n_same_class): idx1 = indxs[i] - idx2 = (idx1 + random.randrange(1, selected_class_n_elements) - ) % selected_class_n_elements - imgs = self._get_images_set( - [selected_class, selected_class], [idx1, idx2], with_aug=with_aug) + idx2 = (idx1 + random.randrange(1, selected_class_n_elements)) % selected_class_n_elements + imgs = self._get_images_set([selected_class, selected_class], [idx1, idx2], with_aug=with_aug) pairs[0][count, :, :, :] = imgs[0] pairs[1][count, :, :, :] = imgs[1] targets[i] = 1 + targets1[i] = t_id + targets2[i] = t_id count += 1 for i in range(n_same_class, self.batch_size): - another_class_idx = ( - selected_class_idx + random.randrange(1, self.n_classes)) % self.n_classes + another_class_idx = (selected_class_idx + random.randrange(1, self.n_classes)) % self.n_classes another_class = self.class_names[another_class_idx] - another_class_n_elements = len(self.indexes[another_class]) + another_class_n_elements = self.n_samples[another_class] idx1 = indxs[i] idx2 = random.randrange(0, another_class_n_elements) - imgs = self._get_images_set( - [selected_class, another_class], [idx1, idx2], with_aug=with_aug) + imgs = self._get_images_set([selected_class, another_class], [idx1, idx2], with_aug=with_aug) pairs[0][count, :, :, :] = imgs[0] pairs[1][count, :, :, :] = imgs[1] targets[i] = 0 + targets1[i] = t_id + targets2[i] = f_id count += 1 - - return pairs, targets + return pairs, {'output_siamese' : targets, + 'output_im1' : targets1, + 'output_im2' : targets2} + # 'model_1' : targets1, + # 'model_1_1' : targets2}) def __getitem__(self, index): return self.get_batch_pairs() diff --git a/embedding_net/model_new.py b/embedding_net/model_new.py index bc8ee57..006263d 100644 --- a/embedding_net/model_new.py +++ b/embedding_net/model_new.py @@ -27,11 +27,17 @@ def __init__(self, params): self.base_model = None self.backbone_model = None + self.model = None self.encoded_training_data = {} def _create_base_model(self): self.base_model, self.backbone_model = get_backbone(**self.params_model) + + # input_image = Input(self.params_model['input_shape']) + # output_base = self.base_model(input_image) + output = Dense(units=1, activation='sigmoid', name='output_img')(self.base_model.layers[-1].output) + self.classification_model = Model(inputs=[self.base_model.layers[0].input],outputs=[output]) def _generate_encodings(self, imgs): encodings = self.base_model.predict(imgs) @@ -78,15 +84,21 @@ def save_encodings(self, encoded_training_data, pickle.dump(encoded_training_data, f) def load_model(self, file_path): - from keras_radam import RAdam - self.model = load_model(file_path, - custom_objects={'contrastive_loss': lac.contrastive_loss, - 'accuracy': lac.accuracy, - 'loss_function': lac.triplet_loss(self.params_generator['margin']), - 'RAdam': RAdam}) + import efficientnet.tfkeras as efn + # from keras_radam import RAdam + # self.model = load_model(file_path, + # custom_objects={'contrastive_loss': lac.contrastive_loss, + # 'accuracy': lac.accuracy, + # 'loss_function': lac.triplet_loss(self.params_generator['margin']), + # 'RAdam': RAdam}) + self.model = load_model(file_path, compile=False) + self.input_shape = list(self.model.inputs[0].shape[1:]) - self.base_model = Model(inputs=[self.model.layers[3].get_input_at(0)], - outputs=[self.model.layers[3].layers[-1].output]) + self.base_model = Model(inputs=[self.model.layers[2].get_input_at(0)], + outputs=[self.model.layers[2].layers[-1].output]) + self.classification_model = Model(inputs=[self.model.layers[3].get_input_at(0)], + outputs=[self.model.layers[-1].output]) + self.classification_model._make_predict_function() self.base_model._make_predict_function() def predict(self, image): @@ -159,10 +171,8 @@ def _create_model_triplet(self): image_encoding_p = self.base_model(input_image_p) image_encoding_n = self.base_model(input_image_n) - merged_vector = concatenate([image_encoding_a, image_encoding_p, image_encoding_n], - axis=-1, name='merged_layer') - self.model = Model(inputs=[input_image_a, input_image_p, input_image_n], - outputs=merged_vector) + merged_vector = concatenate([image_encoding_a, image_encoding_p, image_encoding_n],axis=-1, name='merged_layer') + self.model = Model(inputs=[input_image_a, input_image_p, input_image_n],outputs=merged_vector) print('Base model summary') self.base_model.summary() @@ -170,15 +180,17 @@ def _create_model_triplet(self): print('Whole model summary') self.model.summary() - self.model.compile(loss=lac.triplet_loss(self.params_generator['margin']), - optimizer=self.params_train['optimizer']) - class SiameseNet(EmbeddingNet): def __init__(self, params, training): super().__init__(params) - self._create_model_siamese() + + self.training = training + + if self.training: + self._create_base_model() + self._create_model_siamese() def _create_model_siamese(self): @@ -188,31 +200,29 @@ def _create_model_siamese(self): image_encoding_1 = self.base_model(input_image_1) image_encoding_2 = self.base_model(input_image_2) + Cl_out1 = Lambda(lambda x: x, name='output_im1') + Cl_out2 = Lambda(lambda x: x, name='output_im2') + + classification_output_1 = Cl_out1(self.classification_model(input_image_1)) + classification_output_2 = Cl_out2(self.classification_model(input_image_2)) + if self.params_model['distance_type'] == 'l1': - L1_layer = Lambda( - lambda tensors: K.abs(tensors[0] - tensors[1])) + L1_layer = Lambda(lambda tensors: K.abs(tensors[0] - tensors[1])) distance = L1_layer([image_encoding_1, image_encoding_2]) - prediction = Dense(units=1, activation='sigmoid')(distance) - metric = 'binary_accuracy' + embeddings_output = Dense(units=1, activation='sigmoid', name='output_siamese')(distance) elif self.params_model['distance_type'] == 'l2': - L2_layer = Lambda( - lambda tensors: K.sqrt(K.maximum(K.sum(K.square(tensors[0] - tensors[1]), axis=1, keepdims=True), K.epsilon()))) + L2_layer = Lambda(lambda tensors: K.sqrt(K.maximum(K.sum(K.square(tensors[0] - tensors[1]), axis=1, keepdims=True), K.epsilon()))) distance = L2_layer([image_encoding_1, image_encoding_2]) - prediction = distance - metric = lac.accuracy + embeddings_output = distance - self.model = Model( - inputs=[input_image_1, input_image_2], outputs=prediction) + self.model = Model(inputs=[input_image_1, input_image_2], outputs=[embeddings_output, classification_output_1, classification_output_2]) print('Base model summary') self.base_model.summary() print('Whole model summary') - self.model.summary() - - self.model.compile(loss=lac.contrastive_loss, metrics=[metric], - optimizer=self.params_train['optimizer']) \ No newline at end of file + self.model.summary() \ No newline at end of file diff --git a/get_classification_model.py b/get_classification_model.py new file mode 100644 index 0000000..6459d24 --- /dev/null +++ b/get_classification_model.py @@ -0,0 +1,14 @@ +from tensorflow.keras.models import load_model, Model +import efficientnet.tfkeras as efn +from embedding_net.utils import parse_params +from embedding_net.model_new import SiameseNet + +model_path = '/home/rauf/EmbeddingNet/work_dirs/deepfake_efn_b3/weights/best_deepfake_efn_b3_001_0.578046.hdf5' +cfg_params = parse_params('configs/deepfake_siamese.yml') + +model = SiameseNet(cfg_params, training=True) +model.model.load_weights(model_path, by_name=True) +# model.load_model(model_path) + +model_classification = model.classification_model +model_classification.save("model_classification.h5") diff --git a/train.py b/train.py index 3273223..05d0dfc 100644 --- a/train.py +++ b/train.py @@ -1,14 +1,16 @@ import os import numpy as np -from embedding_net.model_new import EmbeddingNet, TripletNet +from embedding_net.model_new import EmbeddingNet, TripletNet, SiameseNet from tensorflow.keras.callbacks import TensorBoard, LearningRateScheduler from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint from embedding_net.datagenerators import ENDataLoader, SimpleDataGenerator, TripletsDataGenerator, SimpleTripletsDataGenerator, SiameseDataGenerator from embedding_net.utils import parse_params, plot_grapths from embedding_net.backbones import pretrain_backbone_softmax +from embedding_net.losses_and_accuracies import contrastive_loss, triplet_loss, accuracy import argparse from tensorflow import keras import tensorflow as tf +from tensorflow.compat.v1.keras.backend import set_session def parse_args(): @@ -29,7 +31,7 @@ def create_save_folders(params): plots_save_path = os.path.join(work_dir_path, 'plots/') tensorboard_save_path = os.path.join(work_dir_path, 'tf_log/') tensorboard_pretrained_save_path = os.path.join(work_dir_path, 'pretraining_model/tf_log/') - weights_save_file_path = os.path.join(weights_save_path, 'best_' + params['project_name']+'_{epoch:03d}_{loss:03f}' + '.h5') + weights_save_file_path = os.path.join(weights_save_path, 'best_' + params['project_name']+'_{epoch:03d}_{loss:03f}' + '.hdf5') os.makedirs(work_dir_path , exist_ok=True) os.makedirs(weights_save_path, exist_ok=True) @@ -52,11 +54,13 @@ def main(): session = tf.Session(config=config) - keras.backend.set_session(session) + set_session(session) + print('LOAD PARAMETERS') args = parse_args() cfg_params = parse_params(args.config) params_train = cfg_params['train'] + params_model = cfg_params['model'] params_dataloader = cfg_params['dataloader'] params_generator = cfg_params['generator'] @@ -78,6 +82,8 @@ def main(): else: callback_monitor = 'loss' + print('LOADING COMPLETED') + callbacks = [ LearningRateScheduler(lambda x: initial_lr * decay_factor ** np.floor(x/step_size)), @@ -93,13 +99,59 @@ def main(): verbose=1) ] + print('CREATE DATALOADER') data_loader = ENDataLoader(**params_dataloader) - model = TripletNet(cfg_params, training=True) + print('DATALOADER CREATED!') + + val_generator = None + + print('CREATE MODEL AND DATA GENETATORS') + if params_model['mode'] == 'siamese': + model = SiameseNet(cfg_params, training=True) + train_generator = SiameseDataGenerator(class_files_paths=data_loader.train_data, + class_names=data_loader.class_names, + **params_generator) + if data_loader.validate: + val_generator = SiameseDataGenerator(class_files_paths=data_loader.val_data, + class_names=data_loader.class_names, + val_gen = True, + **params_generator) + losses = {'output_siamese' : contrastive_loss, + 'output_im1' : tf.keras.losses.binary_crossentropy, + 'output_im2' : tf.keras.losses.binary_crossentropy} + + metric = {'output_siamese' : accuracy, + 'output_im1' : 'binary_accuracy', + 'output_im2' : 'binary_accuracy'} + # metric = accuracy + else: + model = TripletNet(cfg_params, training=True) + train_generator = TripletsDataGenerator(embedding_model=model.base_model, + class_files_paths=data_loader.train_data, + class_names=data_loader.class_names, + **params_generator) + + if data_loader.validate: + val_generator = SimpleTripletsDataGenerator(data_loader.val_data, + data_loader.class_names, + **params_generator) + losses = triplet_loss(params_generator['margin']) + metric = 'accuracy' + print('DONE') + + print('COMPILE MODEL') + model.model.compile(loss=losses, + optimizer=params_train['optimizer'], + loss_weights = {'output_siamese' : 1, + 'output_im1' : 1, + 'output_im2' : 1}, + metrics=metric) + if args.resume_from is not None: model.load_model(args.resume_from) - model.load_model('work_dirs/deepfake_efn_b3/weights/best_deepfake_efn_b3_001_0.430115.h5') - + # model.load_model('/home/rauf/Desktop/sg_efficientnet-b3__simple_fold_0_005_0.9810.hdf5') + model.base_model.load_weights('/home/rauf/Desktop/sg_efficientnet-b3__simple_fold_0_005_0.9810.hdf5', by_name=True) if 'softmax' in cfg_params: params_softmax = cfg_params['softmax'] params_save_paths = cfg_params['save_paths'] @@ -108,33 +160,6 @@ def main(): params_softmax, params_save_paths) - # def _create_generators(self): - # self.train_generator = SiameseDataGenerator(class_files_paths=self.data_loader.train_data, - # class_names=self.data_loader.class_names, - # **self.params_generator) - # if self.data_loader.validate: - # self.val_generator = SiameseDataGenerator(class_files_paths=self.data_loader.val_data, - # class_names=self.data_loader.class_names, - # **self.params_generator) - - # train_generator = SimpleTripletsDataGenerator(class_files_paths=data_loader.train_data, - # class_names=data_loader.class_names, - # **params_generator) - # checkpoints_load_name = 'work_dirs/bengali_efn_b5/pretraining_model/weights/best_efficientnet-b5.hdf5' - # model.base_model.load_weights(checkpoints_load_name, by_name=True) - train_generator = TripletsDataGenerator(embedding_model=model.base_model, - class_files_paths=data_loader.train_data, - class_names=data_loader.class_names, - **params_generator) - - - if data_loader.validate: - val_generator = SimpleTripletsDataGenerator(data_loader.val_data, - data_loader.class_names, - **params_generator) - else: - val_generator = None - history = model.model.fit_generator(train_generator, validation_data=val_generator, epochs=params_train['n_epochs'],