add optional validation

.gitignore

@@ -115,3 +115,4 @@ plots/
 sub.csv
 core
 work_dirs/
+*.csv

configs/plates_resnet18.yml

@@ -2,7 +2,7 @@ input_shape :  [128, 128, 3]
 encodings_len: 128
 margin: 0.4
 mode : 'triplet'
-distance_type : 'l1'
+distance_type : 'l2'
 backbone : 'resnet18'
 backbone_weights : 'imagenet'
 freeze_backbone : True
@@ -11,6 +11,7 @@ min_n_obj_per_class : 0
 select_max_n_obj_per_class : 30
 max_n_obj_per_class : 10000
 embeddings_normalization: True
+to_validate : False
 
 # optimizer parameters
 optimizer : 'radam'
@@ -25,7 +26,7 @@ val_batch_size : 16
 val_steps : 10
 negatives_selection_mode : 'semihard'
 mining_n_classes: 2
-mining_n_samples: 5
+mining_n_samples: 7
 
 # softmax pretraining parameters
 softmax_pretraining : True
@@ -36,7 +37,6 @@ softmax_steps_per_epoch : 100
 softmax_epochs : 100
 softmax_is_binary: True
 
-#paths
 # paths
 work_dir : 'work_dirs/plates/'
 dataset_path : '/home/rauf/datasets/plates/plates_splitted/'

configs/plates_resnext50.yml

@@ -1,8 +1,8 @@
 input_shape :  [128, 128, 3]
 encodings_len: 128
-margin: 0.3
+margin: 0.4
 mode : 'triplet'
-distance_type : 'l1'
+distance_type : 'l2'
 backbone : 'resnext50'
 backbone_weights : 'imagenet'
 freeze_backbone : True
@@ -11,32 +11,32 @@ min_n_obj_per_class : 0
 select_max_n_obj_per_class : 30
 max_n_obj_per_class : 10000
 embeddings_normalization: True
+to_validate : False
 
 # optimizer parameters
 optimizer : 'radam'
-learning_rate : 0.00001
+learning_rate : 0.0001
 decay_factor : 0.99999
 step_size : 1
 
 # embeddings learning training parameters
 n_epochs : 1000
 n_steps_per_epoch : 200
-val_batch_size : 4
+val_batch_size : 16
 val_steps : 10
 negatives_selection_mode : 'semihard'
 mining_n_classes: 2
-mining_n_samples: 3
+mining_n_samples: 5
 
 # softmax pretraining parameters
 softmax_pretraining : True
-softmax_batch_size_train : 16
-softmax_batch_size_val : 8
-softmax_val_steps : 1
-softmax_steps_per_epoch : 10
+softmax_batch_size_train : 8
+softmax_batch_size_val : 4
+softmax_val_steps : 50
+softmax_steps_per_epoch : 200
 softmax_epochs : 100
 softmax_is_binary: True
 
-#paths
 # paths
 work_dir : 'work_dirs/plates_resnext50/'
 dataset_path : '/home/rauf/datasets/plates/plates_splitted/'

embedding_net/augmentations.py

@@ -34,8 +34,12 @@ def get_aug(name='default', input_shape=[48, 48, 3]):
             A.VerticalFlip(p=0.5),
             A.RandomBrightness(limit=0.2, p=0.3),
             A.RandomContrast(limit=0.2, p=0.3),
+            # A.Rotate(limit=360, p=0.9),
             A.RandomRotate90(p=0.3),
-            A.HueSaturationValue(hue_shift_limit=(-30,30), sat_shift_limit=(-10,10), val_shift_limit=(-10,10), p=0.5),
+            A.HueSaturationValue(hue_shift_limit=(-50,50), 
+                                 sat_shift_limit=(-15,15), 
+                                 val_shift_limit=(-15,15), 
+                                 p=0.5),
 #             A.Blur(blur_limit=(5,7), p=0.3),
             A.GaussNoise(var_limit=(10, 50), p=0.3),
             A.CenterCrop(p=1, height=2*input_shape[1]//3, width=2*input_shape[0]//3),

embedding_net/data_loader.py

@@ -17,25 +17,24 @@ def __init__(self, dataset_path,
                        augmentations=None, 
                        min_n_obj_per_class = None,
                        select_max_n_obj_per_class = None, 
-                       max_n_obj_per_class = None, 
-                       data_subsets=['train', 'val']):
+                       max_n_obj_per_class = None):
         self.dataset_path = dataset_path
-        self.data_subsets = data_subsets
+        self.data_subsets = [d.split('/')[-1] for d in os.listdir(self.dataset_path) if os.path.isdir(os.path.join(self.dataset_path, d))]
         self.images_paths = {}
         self.images_labels = {}
         self.input_shape = input_shape
         self.augmentations = augmentations
         self.min_n_obj_per_class = min_n_obj_per_class if min_n_obj_per_class else 0
         self.select_max_n_obj_per_class = select_max_n_obj_per_class if select_max_n_obj_per_class else 1e10
         self.max_n_obj_per_class = max_n_obj_per_class if max_n_obj_per_class else 1e10
-        self.current_idx = {d: 0 for d in data_subsets}
+        self.current_idx = {d: 0 for d in self.data_subsets}
         self._load_images_paths()
         self.classes = {
-            s: sorted(list(set(self.images_labels[s]))) for s in data_subsets}
-        self.n_classes = {s: len(self.classes[s]) for s in data_subsets}
-        self.n_samples = {d: len(self.images_paths[d]) for d in data_subsets}
+            s: sorted(list(set(self.images_labels[s]))) for s in self.data_subsets}
+        self.n_classes = {s: len(self.classes[s]) for s in self.data_subsets}
+        self.n_samples = {d: len(self.images_paths[d]) for d in self.data_subsets}
         self.indexes = {d: {cl: np.where(np.array(self.images_labels[d]) == cl)[
-            0] for cl in self.classes[d]} for d in data_subsets}
+            0] for cl in self.classes[d]} for d in self.data_subsets}
 
     def _load_images_paths(self):
         skip_list = ['train','val','test']

embedding_net/model.py

@@ -89,7 +89,12 @@ def pretrain_backbone_softmax(self):
         epochs = self.cfg_params['softmax_epochs']
 
         train_generator = self.data_loader.generate(batch_size_train,is_binary=is_binary, mode='simple', s="train")
-        val_generator = self.data_loader.generate(batch_size_val,is_binary=is_binary, mode='simple', s="val")
+        if 'val' in self.data_loader.data_subsets and self.cfg_params['to_validate']:
+            val_generator = self.data_loader.generate(batch_size_val,is_binary=is_binary, mode='simple', s="val")
+            checkpoint_callback_monitor = 'val_loss'
+        else:
+            val_generator = None
+            checkpoint_callback_monitor = 'loss'
 
         tensorboard_save_path = os.path.join(
             self.cfg_params['work_dir'], 'tf_log/pretraining_model/')
@@ -105,12 +110,17 @@ def pretrain_backbone_softmax(self):
         callbacks = [
             LearningRateScheduler(lambda x: initial_lr *
                                 decay_factor ** np.floor(x/step_size)),
-            ReduceLROnPlateau(monitor='val_loss', factor=0.1,
+            ReduceLROnPlateau(monitor=checkpoint_callback_monitor, factor=0.1,
                             patience=20, verbose=1),
-            EarlyStopping(patience=10, verbose=1, restore_best_weights=True),
+            EarlyStopping(monitor=checkpoint_callback_monitor,
+                          patience=10, 
+                          verbose=1, 
+                          restore_best_weights=True),
             TensorBoard(log_dir=tensorboard_save_path),
             ModelCheckpoint(filepath=weights_save_file,
-                            verbose=1, monitor='val_loss', save_best_only=True)
+                            verbose=1, 
+                            monitor=checkpoint_callback_monitor, 
+                            save_best_only=True)
         ]
 
         history = model.fit_generator(train_generator,
@@ -208,23 +218,38 @@ def validate_on_batch(self, batch_size=8, s="val"):
             pairs, targets)
         return val_loss, val_accuracy
 
-    def train_generator(self, steps_per_epoch, epochs, callbacks=[], val_steps=100, with_val=True, batch_size=8, verbose=1):
+    def train_generator(self, 
+                        steps_per_epoch, 
+                        epochs, 
+                        callbacks=[], 
+                        val_steps=100,  
+                        batch_size=8, 
+                        verbose=1):
 
         train_generator = self.data_loader.generate(
             batch_size, mode=self.mode, s="train")
-        val_generator = self.data_loader.generate(
-            batch_size, mode=self.mode, s="val")
+
+        if 'val' in self.data_loader.data_subsets and self.cfg_params['to_validate']:
+            val_generator = self.data_loader.generate(
+                batch_size, mode=self.mode, s="val")
+        else:
+            val_generator = None
 
-        history = self.model.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=epochs,
-                                           verbose=verbose, validation_data=val_generator, validation_steps=val_steps, callbacks=callbacks)
+        history = self.model.fit_generator(train_generator, 
+                                           steps_per_epoch=steps_per_epoch, 
+                                           epochs=epochs,
+                                           verbose=verbose, 
+                                           validation_data=val_generator, 
+                                           validation_steps=val_steps, 
+                                           callbacks=callbacks)
 
         return history
 
     def train_generator_mining(self,
                                steps_per_epoch,
-                               epochs, callbacks=[],
+                               epochs, 
+                               callbacks=[],
                                val_steps=100,
-                               with_val=True,
                                n_classes=4,
                                n_samples=4,
                                val_batch=8,
@@ -233,8 +258,12 @@ def train_generator_mining(self,
 
         train_generator = self.data_loader.generate_mining(
             self.base_model, n_classes, n_samples, margin=self.margin, negative_selection_mode=negative_selection_mode, s="train")
-        val_generator = self.data_loader.generate(
-            val_batch, mode=self.mode, s="val")
+
+        if 'val' in self.data_loader.data_subsets and self.cfg_params['to_validate']:
+            val_generator = self.data_loader.generate(
+                val_batch, mode=self.mode, s="val")
+        else:
+            val_generator = None
 
         history = self.model.fit_generator(train_generator,
                                            steps_per_epoch=steps_per_epoch,

train.py

@@ -48,15 +48,22 @@ def main():
     decay_factor = cfg_params['decay_factor']
     step_size = cfg_params['step_size']
 
+    if cfg_params['to_validate']:
+        callback_monitor = 'val_loss'
+    else:
+        callback_monitor = 'loss'
+
     callbacks = [
         LearningRateScheduler(lambda x: initial_lr *
                               decay_factor ** np.floor(x/step_size)),
-        ReduceLROnPlateau(monitor='val_loss', factor=0.1,
+        ReduceLROnPlateau(monitor=callback_monitor, factor=0.1,
                           patience=4, verbose=1),
-        EarlyStopping(patience=10, verbose=1),
+        EarlyStopping(monitor=callback_monitor,
+                      patience=10, 
+                      verbose=1),
         TensorBoard(log_dir=tensorboard_save_path),
         ModelCheckpoint(filepath=weights_save_file,
-                        verbose=1, monitor='val_loss', save_best_only=True)
+                        verbose=1, monitor=callback_monitor, save_best_only=True)
     ]
 
     history = model.train_generator_mining(steps_per_epoch=cfg_params['n_steps_per_epoch'],
@@ -79,10 +86,10 @@ def main():
                                  max_num_samples_of_each_class=cfg_params['max_num_samples_of_each_class'],
                                  knn_k=cfg_params['knn_k'],
                                  shuffle=True)
-
-        model_accuracies = model.calculate_prediction_accuracy()
-        print('Model top1 accuracy on validation set: {}'.format(model_accuracies['top1']))
-        print('Model top5 accuracy on validation set: {}'.format(model_accuracies['top5']))
+        if cfg_params['to_validate']:
+            model_accuracies = model.calculate_prediction_accuracy()
+            print('Model top1 accuracy on validation set: {}'.format(model_accuracies['top1']))
+            print('Model top5 accuracy on validation set: {}'.format(model_accuracies['top5']))
 
 
 if __name__ == '__main__':