add some changes

configs/deepfake.yml

@@ -2,7 +2,7 @@ MODEL:
   input_shape :  [224, 224, 3]
   encodings_len: 256
   mode : 'triplet'
-  distance_type : 'l1'
+  distance_type : 'l2'
   backbone_name : 'efficientnet-b3'
   backbone_weights : 'noisy-student'
   freeze_backbone : False

configs/deepfake_siamese.yml

@@ -0,0 +1,59 @@
+MODEL:
+  input_shape :  [224, 224, 3]
+  encodings_len: 256
+  mode : 'siamese'
+  distance_type : 'l1'
+  backbone_name : 'efficientnet-b3'
+  backbone_weights : 'noisy-student'
+  freeze_backbone : False
+  embeddings_normalization: True
+
+DATALOADER:
+  dataset_path : '/home/rauf/datasets/aaaa/deepfake/'
+  csv_file : 
+  image_id_column : 
+  label_column : 
+  validate : True
+  val_ratio : 0.2
+
+GENERATOR:
+  batch_size : 5
+  n_batches : 5000
+  n_batches_val : 500
+  augmentations : 'deepfake'
+
+TRAIN:
+  # optimizer parameters
+  optimizer : 'radam'
+  learning_rate : 0.00016
+  decay_factor : 0.95
+  step_size : 1
+
+  # embeddings learning training parameters
+  n_epochs : 1000
+
+  # plot training history
+  plot_history : True
+
+# SOFTMAX_PRETRAINING:
+#   # softmax pretraining parameters
+#   optimizer : 'radam'
+#   learning_rate : 0.0001
+#   decay_factor : 0.99
+#   step_size : 1
+
+#   batch_size : 16
+#   val_steps : 200
+#   steps_per_epoch : 1000
+#   n_epochs : 50
+
+SAVE_PATHS:
+  project_name : 'deepfake_efn_b3'
+  work_dir : 'work_dirs/'
+
+ENCODINGS:
+  # encodings parameters
+  save_encodings : True
+  centers_only: False
+  max_num_samples_of_each_class : 30
+  knn_k : 1

embedding_net/datagenerators.py

@@ -57,53 +57,60 @@ def _load_from_dataframe(self, csv_file, image_id_column, label_column):
         for class_name in self.class_names:
             image_names = dataframe.loc[dataframe[label_column] == class_name][image_id_column]
             image_paths = [os.path.join(self.dataset_path, f) for f in image_names]
-            self.class_files_paths[class_name] = image_paths       
+            self.class_files_paths[class_name] = image_paths      
 
     def _load_from_directory(self):
         self.class_names = [f.name for f in os.scandir(self.dataset_path) if f.is_dir()]
         class_dir_paths = [f.path for f in os.scandir(self.dataset_path) if f.is_dir()]
-        
+
         for class_name, class_dir_path in zip(self.class_names, class_dir_paths):
             subdirs = [f.path for f in os.scandir(class_dir_path) if f.is_dir()]
-            self.class_files_paths[class_name] = []
+            # self.class_files_paths[class_name] = []
+            temp_list = []
             print(class_dir_path)
             if len(subdirs)>0:
                 for subdir in subdirs:
                     class_image_paths = [f.path for f in os.scandir(subdir) if f.is_file() and
                                         (f.name.endswith('.jpg') or
                                         f.name.endswith('.png') and 
                                         not f.name.startswith('._'))]
-                    for class_image_path in class_image_paths:
-                        self.class_files_paths[class_name].append(class_image_path)
+                    temp_list.extend(class_image_paths)
             else:  
                 class_image_paths = [f.path for f in os.scandir(class_dir_path) if f.is_file() and
                                         (f.name.endswith('.jpg') or
                                         f.name.endswith('.png') and 
                                         not f.name.startswith('._'))]
-                for class_image_path in class_image_paths:
-                        self.class_files_paths[class_name].append(class_image_path)  
+                temp_list.extend(class_image_paths)
+            self.class_files_paths[class_name] = temp_list 
 
 
 class ENDataGenerator(Sequence):
     def __init__(self, class_files_paths,
                        class_names,
+                       val_gen = False,
                        input_shape=None,
                        batch_size = 32,
                        n_batches = 10, 
+                       n_batches_val = 10,
                        augmentations=None):
 
         self.input_shape = input_shape
         self.augmentations = augmentations
         self.batch_size = batch_size
         self.n_batches = n_batches
+        self.n_batches_val = n_batches_val
+        self.val_gen = val_gen
         self.class_files_paths = class_files_paths
         self.class_names = class_names
 
         self.n_classes = len(self.class_names)
         self.n_samples = {k: len(v) for k, v in self.class_files_paths.items()}
 
     def __len__(self):
-        return self.n_batches
+        if self.val_gen:
+            return self.n_batches_val
+        else:
+            return self.n_batches
 
     def __getitem__(self, index):
         pass       
@@ -119,7 +126,7 @@ def _get_images_set(self, clsss, idxs, with_aug=True):
         if with_aug:
             imgs = [self.augmentations(image=img)['image'] for img in imgs]
 
-        return np.array(imgs)
+        return np.array(imgs)/255.
 
 
 class TripletsDataGenerator(ENDataGenerator):
@@ -289,60 +296,75 @@ class SiameseDataGenerator(ENDataGenerator):
 
     def __init__(self, class_files_paths,
                        class_names,
+                       val_gen = False,
                        input_shape=None,
                        batch_size = 32,
-                       n_batches = 10, 
+                       n_batches = 10,
+                       n_batches_val = 10, 
                        augmentations=None):
 
         super().__init__(class_files_paths=class_files_paths, 
-                         class_names=class_names, 
+                         class_names=class_names,
+                         val_gen = False, 
                          input_shape=input_shape, 
                          batch_size=batch_size, 
-                         n_batches=n_batches, 
+                         n_batches=n_batches,
+                         n_batches_val = 10, 
                          augmentations=augmentations)
 
     def get_batch_pairs(self):
         pairs = [np.zeros((self.batch_size, self.input_shape[0], self.input_shape[1], 3)), np.zeros(
             (self.batch_size, self.input_shape[0], self.input_shape[1], 3))]
-        targets = np.zeros((self.batch_size,))
+        targets = np.zeros((self.batch_size, ))
+        targets1 = np.zeros((self.batch_size, ))
+        targets2 = np.zeros((self.batch_size, ))
+        # targets = []
 
         n_same_class = self.batch_size // 2
 
         selected_class_idx = random.randrange(0, self.n_classes)
         selected_class = self.class_names[selected_class_idx]
-        selected_class_n_elements = len(self.indexes[selected_class])
+        selected_class_n_elements = self.n_samples[selected_class] 
 
-        indxs = np.random.randint(
-            selected_class_n_elements, size=self.batch_size)
+        if selected_class == 'real':
+            t_id = 0
+            f_id = 1
+        else:
+            t_id = 1
+            f_id = 0
+        indxs = np.random.randint(selected_class_n_elements, size=self.batch_size)
 
         with_aug = self.augmentations
         count = 0
         for i in range(n_same_class):
             idx1 = indxs[i]
-            idx2 = (idx1 + random.randrange(1, selected_class_n_elements)
-                    ) % selected_class_n_elements
-            imgs = self._get_images_set(
-                [selected_class, selected_class], [idx1, idx2], with_aug=with_aug)
+            idx2 = (idx1 + random.randrange(1, selected_class_n_elements)) % selected_class_n_elements
+            imgs = self._get_images_set([selected_class, selected_class], [idx1, idx2], with_aug=with_aug)
             pairs[0][count, :, :, :] = imgs[0]
             pairs[1][count, :, :, :] = imgs[1]
             targets[i] = 1
+            targets1[i] = t_id
+            targets2[i] = t_id
             count += 1
 
         for i in range(n_same_class, self.batch_size):
-            another_class_idx = (
-                selected_class_idx + random.randrange(1, self.n_classes)) % self.n_classes
+            another_class_idx = (selected_class_idx + random.randrange(1, self.n_classes)) % self.n_classes
             another_class = self.class_names[another_class_idx]
-            another_class_n_elements = len(self.indexes[another_class])
+            another_class_n_elements = self.n_samples[another_class]
             idx1 = indxs[i]
             idx2 = random.randrange(0, another_class_n_elements)
-            imgs = self._get_images_set(
-                [selected_class, another_class], [idx1, idx2], with_aug=with_aug)
+            imgs = self._get_images_set([selected_class, another_class], [idx1, idx2], with_aug=with_aug)
             pairs[0][count, :, :, :] = imgs[0]
             pairs[1][count, :, :, :] = imgs[1]
             targets[i] = 0
+            targets1[i] = t_id
+            targets2[i] = f_id
             count += 1
-
-        return pairs, targets
+        return pairs, {'output_siamese' : targets, 
+                       'output_im1' : targets1, 
+                       'output_im2' : targets2}
+                        # 'model_1' : targets1,
+                        # 'model_1_1' : targets2})
 
     def __getitem__(self, index):
         return self.get_batch_pairs()

embedding_net/model_new.py

@@ -27,11 +27,17 @@ def __init__(self,  params):
 
         self.base_model = None
         self.backbone_model = None
+        self.model = None
 
         self.encoded_training_data = {}
 
     def _create_base_model(self):
         self.base_model, self.backbone_model = get_backbone(**self.params_model)
+
+        # input_image = Input(self.params_model['input_shape'])
+        # output_base = self.base_model(input_image)
+        output = Dense(units=1, activation='sigmoid', name='output_img')(self.base_model.layers[-1].output)
+        self.classification_model = Model(inputs=[self.base_model.layers[0].input],outputs=[output])
 
     def _generate_encodings(self, imgs):
         encodings = self.base_model.predict(imgs)
@@ -78,15 +84,21 @@ def save_encodings(self, encoded_training_data,
             pickle.dump(encoded_training_data, f)
 
     def load_model(self, file_path):
-        from keras_radam import RAdam
-        self.model = load_model(file_path,
-                                custom_objects={'contrastive_loss': lac.contrastive_loss,
-                                                'accuracy': lac.accuracy,
-                                                'loss_function': lac.triplet_loss(self.params_generator['margin']),
-                                                'RAdam': RAdam})
+        import efficientnet.tfkeras as efn
+        # from keras_radam import RAdam
+        # self.model = load_model(file_path,
+        #                         custom_objects={'contrastive_loss': lac.contrastive_loss,
+        #                                         'accuracy': lac.accuracy,
+        #                                         'loss_function': lac.triplet_loss(self.params_generator['margin']),
+        #                                         'RAdam': RAdam})
+        self.model = load_model(file_path, compile=False)
+
         self.input_shape = list(self.model.inputs[0].shape[1:])
-        self.base_model = Model(inputs=[self.model.layers[3].get_input_at(0)],
-                                outputs=[self.model.layers[3].layers[-1].output])
+        self.base_model = Model(inputs=[self.model.layers[2].get_input_at(0)],
+                                outputs=[self.model.layers[2].layers[-1].output])
+        self.classification_model = Model(inputs=[self.model.layers[3].get_input_at(0)],
+                                outputs=[self.model.layers[-1].output])
+        self.classification_model._make_predict_function()
         self.base_model._make_predict_function()
 
     def predict(self, image):
@@ -159,26 +171,26 @@ def _create_model_triplet(self):
         image_encoding_p = self.base_model(input_image_p)
         image_encoding_n = self.base_model(input_image_n)
 
-        merged_vector = concatenate([image_encoding_a, image_encoding_p, image_encoding_n],
-                                    axis=-1, name='merged_layer')
-        self.model = Model(inputs=[input_image_a, input_image_p, input_image_n],
-                           outputs=merged_vector)
+        merged_vector = concatenate([image_encoding_a, image_encoding_p, image_encoding_n],axis=-1, name='merged_layer')
+        self.model = Model(inputs=[input_image_a, input_image_p, input_image_n],outputs=merged_vector)
 
         print('Base model summary')
         self.base_model.summary()
 
         print('Whole model summary')
         self.model.summary()
 
-        self.model.compile(loss=lac.triplet_loss(self.params_generator['margin']), 
-                                                 optimizer=self.params_train['optimizer'])
-
 
 class SiameseNet(EmbeddingNet):
 
     def __init__(self, params, training):
         super().__init__(params)
-        self._create_model_siamese()
+
+        self.training = training
+
+        if self.training:
+            self._create_base_model()
+            self._create_model_siamese()
 
     def _create_model_siamese(self):
 
@@ -188,31 +200,29 @@ def _create_model_siamese(self):
         image_encoding_1 = self.base_model(input_image_1)
         image_encoding_2 = self.base_model(input_image_2)
 
+        Cl_out1 = Lambda(lambda x: x, name='output_im1')
+        Cl_out2 = Lambda(lambda x: x, name='output_im2')
+
+        classification_output_1 = Cl_out1(self.classification_model(input_image_1))
+        classification_output_2 = Cl_out2(self.classification_model(input_image_2))
+
         if self.params_model['distance_type'] == 'l1':
-            L1_layer = Lambda(
-                lambda tensors: K.abs(tensors[0] - tensors[1]))
+            L1_layer = Lambda(lambda tensors: K.abs(tensors[0] - tensors[1]))
             distance = L1_layer([image_encoding_1, image_encoding_2])
 
-            prediction = Dense(units=1, activation='sigmoid')(distance)
-            metric = 'binary_accuracy'
+            embeddings_output = Dense(units=1, activation='sigmoid', name='output_siamese')(distance)
 
         elif self.params_model['distance_type'] == 'l2':
 
-            L2_layer = Lambda(
-                lambda tensors: K.sqrt(K.maximum(K.sum(K.square(tensors[0] - tensors[1]), axis=1, keepdims=True), K.epsilon())))
+            L2_layer = Lambda(lambda tensors: K.sqrt(K.maximum(K.sum(K.square(tensors[0] - tensors[1]), axis=1, keepdims=True), K.epsilon())))
             distance = L2_layer([image_encoding_1, image_encoding_2])
 
-            prediction = distance
-            metric = lac.accuracy
+            embeddings_output = distance
 
-        self.model = Model(
-            inputs=[input_image_1, input_image_2], outputs=prediction)
+        self.model = Model(inputs=[input_image_1, input_image_2], outputs=[embeddings_output, classification_output_1, classification_output_2])
 
         print('Base model summary')
         self.base_model.summary()
 
         print('Whole model summary')
-        self.model.summary()
-
-        self.model.compile(loss=lac.contrastive_loss, metrics=[metric],
-                           optimizer=self.params_train['optimizer'])
+        self.model.summary()

get_classification_model.py

@@ -0,0 +1,14 @@
+from tensorflow.keras.models import load_model, Model
+import efficientnet.tfkeras as efn
+from embedding_net.utils import parse_params
+from embedding_net.model_new import SiameseNet
+
+model_path = '/home/rauf/EmbeddingNet/work_dirs/deepfake_efn_b3/weights/best_deepfake_efn_b3_001_0.578046.hdf5'
+cfg_params = parse_params('configs/deepfake_siamese.yml')
+
+model = SiameseNet(cfg_params, training=True)
+model.model.load_weights(model_path, by_name=True)
+# model.load_model(model_path)
+
+model_classification = model.classification_model
+model_classification.save("model_classification.h5")