add train_generator instead of own training function

.gitignore

@@ -109,3 +109,4 @@ tf_log/
 to_test.ipynb
 encodings/
 weights/
+plots/

model.py

@@ -5,14 +5,15 @@
 import tensorflow as tf
 import pickle
 import cv2
+import random
 from keras.models import Model
 from keras import optimizers
 from keras.regularizers import l2
 from keras.utils import plot_model
 from keras.layers import Dense, Input, Lambda, Dropout, Flatten
 from keras.layers import Conv2D, MaxPool2D
 from classification_models import Classifiers
-from keras.callbacks import TensorBoard
+
 
 
 class SiameseNet:
@@ -23,32 +24,50 @@ class SiameseNet:
 
     """
 
-    def __init__(self, input_shape, image_loader, mode='l1', backbone='resnet50', optimizer=optimizers.Adam(lr=1e-4), tensorboard_log_path='tf_log/', weights_save_path='weights/'):
+    def __init__(self, input_shape, image_loader, mode='l1', backbone='resnet50',
+                 backbone_weights = 'imagenet',
+                 optimizer=optimizers.Adam(lr=1e-4), tensorboard_log_path='tf_log/',
+                 weights_save_path='weights/', plots_path='plots/', encodings_path='encodings/',
+                 project_name='', freeze_backbone=True):
         self.input_shape = input_shape
         self.backbone = backbone
+        self.backbone_weights = backbone_weights
         self.mode = mode
+        self.project_name = project_name
         self.optimizer = optimizer
         self.model = []
         self.base_model = []
-        self._create_model()
-        self.data_loader = image_loader
-        self.encoded_training_data = {}
-        if tensorboard_log_path:
-            os.makedirs(tensorboard_log_path, exist_ok=True)
+        self.l_model = []
+        self.freeze_backbone = freeze_backbone
+
+        self.encodings_path = os.path.join(encodings_path, project_name)
+        os.makedirs(self.encodings_path, exist_ok=True)
+        self.plots_path = os.path.join(plots_path, project_name)
+        self.tensorboard_log_path = os.path.join(
+            tensorboard_log_path, project_name)
+        if self.plots_path:
+            os.makedirs(self.plots_path, exist_ok=True)
+        if self.tensorboard_log_path:
+            os.makedirs(self.tensorboard_log_path, exist_ok=True)
         self.tensorboard_callback = TensorBoard(
-            tensorboard_log_path) if tensorboard_log_path else None
+            self.tensorboard_log_path) if tensorboard_log_path else None
         if self.tensorboard_callback:
             events_files_list = glob.glob(
-                os.path.join(tensorboard_log_path, 'events*'))
+                os.path.join(self.tensorboard_log_path, 'events*'))
             for event_file in events_files_list:
                 try:
                     os.remove(event_file)
                 except:
                     print("Error while deleting file : ", event_file)
             self.tensorboard_callback.set_model(self.model)
-        self.weights_save_path = weights_save_path
-        if weights_save_path:
-            os.makedirs(weights_save_path, exist_ok=True)
+        self.weights_save_path = os.path.join(
+            weights_save_path, self.project_name)
+        if self.weights_save_path:
+            os.makedirs(self.weights_save_path, exist_ok=True)
+
+        self._create_model()
+        self.data_loader = image_loader
+        self.encoded_training_data = {}
 
     def _create_model(self):
 
@@ -69,25 +88,26 @@ def _create_model(self):
             x = Conv2D(256, (4, 4), activation='relu',
                        kernel_regularizer=l2(2e-4))(x)
             x = Flatten()(x)
-            encoded_output = Dense(2048, activation='sigmoid',
+            encoded_output = Dense(4096, activation='sigmoid',
                                    kernel_regularizer=l2(1e-3))(x)
             self.base_model = Model(
                 inputs=[input_image], outputs=[encoded_output])
         else:
             classifier, preprocess_input = Classifiers.get(self.backbone)
             backbone_model = classifier(
-                input_shape=self.input_shape, weights='imagenet', include_top=False)
+                input_shape=self.input_shape, weights=self.backbone_weights, include_top=False)
 
-            for layer in backbone_model.layers:
-                layer.trainable = False
+            if self.freeze_backbone:
+                for layer in backbone_model.layers:
+                    layer.trainable = False
 
             after_backbone = backbone_model.output
             x = Flatten()(after_backbone)
-            x = Dense(512, activation="relu")(x)
-            x = Dropout(0.1)(x)
-            x = Dense(256, activation="relu")(x)
-            x = Dropout(0.1)(x)
-            encoded_output = Dense(256, activation="relu")(x)
+            # x = Dense(512, activation="relu")(x)
+            # x = Dropout(0.5)(x)
+            # x = Dense(512, activation="relu")(x)
+            # x = Dropout(0.5)(x)
+            encoded_output = Dense(4096, activation="relu")(x)
 
             self.base_model = Model(
                 inputs=[backbone_model.input], outputs=[encoded_output])
@@ -102,6 +122,7 @@ def _create_model(self):
 
             prediction = Dense(units=1, activation='sigmoid')(distance)
             metric = 'binary_accuracy'
+
         elif self.mode == 'l2':
 
             L2_layer = Lambda(
@@ -110,10 +131,12 @@ def _create_model(self):
 
             prediction = distance
             metric = self.accuracy
+
+        # self.l_model = Model(inputs=[image_encoding_1, image_encoding_2], outputs=[prediction])
         self.model = Model(
             inputs=[input_image_1, input_image_2], outputs=prediction)
 
-        plot_model(self.model, to_file='plots/model.png')
+        plot_model(self.model, to_file='{}model.png'.format(self.plots_path))
         print('BASE MODEL SUMMARY')
         self.base_model.summary()
 
@@ -160,57 +183,14 @@ def validate_on_batch(self, batch_size=8, s="val"):
             pairs, targets)
         return val_loss, val_accuracy
 
-    def train(self, steps_per_epoch, epochs, val_steps=100, with_val=True, batch_size=8, verbose=1):
-        generator_train = self.data_loader.generate(batch_size, 'train')
-        train_accuracies_epochs = []
-        train_losses_epochs = []
-        val_accuracies_epochs = []
-        val_losses_epochs = []
-        best_val_accuracy = 0.0
-        current_accuracy = 0.0
-        tensorboard_names = ['train_loss', 'train_acc', 'val_loss', 'val_acc']
-        for j in range(epochs):
-            train_accuracies_it = []
-            train_losses_it = []
-            for i in range(steps_per_epoch):
-                pairs, targets = next(generator_train)
-                train_loss_it, train_accuracy_it = self.model.train_on_batch(
-                    pairs, targets)
-                train_accuracies_it.append(train_accuracy_it)
-                train_losses_it.append(train_loss_it)
-            train_loss_epoch = sum(train_losses_it) / len(train_losses_it)
-            train_accuracy_epoch = sum(
-                train_accuracies_it) / len(train_accuracies_it)
-            train_accuracies_epochs.append(train_accuracy_epoch)
-            train_losses_epochs.append(train_loss_epoch)
-
-            if with_val:
-                val_loss, val_accuracy = self.validate(
-                    number_of_comparisons=val_steps)
-                val_accuracies_epochs.append(val_accuracy)
-                val_losses_epochs.append(val_loss)
-                if verbose:
-                    print('[Epoch {}] train_loss: {} , train_acc: {}, val_loss: {} , val_acc: {}'.format(
-                        j, train_loss_epoch, train_accuracy_epoch, val_loss, val_accuracy))
-                logs = [train_loss_epoch, train_accuracy_epoch,
-                        val_loss, val_accuracy]
-
-                if val_accuracy > best_val_accuracy and self.weights_save_path:
-                    best_val_accuracy = val_accuracy
-                    self.base_model.save(
-                        "{}best_model.h5".format(self.weights_save_path))
-            else:
-                if verbose:
-                    print('[Epoch {}] train_loss: {} , train_acc: {}'.format(
-                        j, train_loss_epoch, train_accuracy_epoch))
-                tensorboard_names = tensorboard_names[:2]
-                logs = [train_loss_epoch, train_accuracy_epoch]
-            if self.tensorboard_callback:
-                self.write_log(tensorboard_names, logs, j)
-        if with_val:
-            return train_losses_epochs, train_accuracies_epochs, val_losses_epochs, val_accuracies_epochs
-        else:
-            return train_losses_epochs, train_accuracies_epochs
+    def train_generator(self, steps_per_epoch, epochs, callbacks = [], val_steps=100, with_val=True, batch_size=8, verbose=1):
+
+        train_generator = self.data_loader.generate(batch_size, s="train")
+        val_generator = self.data_loader.generate(batch_size, s="val")
+
+        history = self.model.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=epochs,
+                                 verbose=verbose, validation_data = val_generator, validation_steps = val_steps, callbacks=callbacks)
+        return history
 
     def validate(self, number_of_comparisons=100, batch_size=4, s="val"):
         generator = self.data_loader.generate(batch_size, s=s)
@@ -237,20 +217,30 @@ def _generate_encoding(self, img_path):
         encoding = self.base_model.predict(np.expand_dims(img, axis=0))
         return encoding
 
-    def generate_encodings(self, encodings_path='encodings/', save_file_name='encodings.pkl'):
+    def generate_encodings(self, save_file_name='encodings.pkl', max_num_samples_of_each_classes=10, shuffle = True):
         data_paths, data_labels, data_encodings = [], [], []
+        classes_counter = {}
+
+        if shuffle:
+            c = list(zip(self.data_loader.images_paths['train'], self.data_loader.images_labels['train']))
+            random.shuffle(c)
+            self.data_loader.images_paths['train'], self.data_loader.images_labels['train'] = zip(*c)
 
         for img_path, img_label in zip(self.data_loader.images_paths['train'],
                                        self.data_loader.images_labels['train']):
-            data_paths.append(img_path)
-            data_labels.append(img_label)
-            data_encodings.append(self._generate_encoding(img_path))
+            if img_label not in classes_counter:
+                classes_counter[img_label] = 0
+            classes_counter[img_label] += 1
+            if classes_counter[img_label] < max_num_samples_of_each_classes:
+                data_paths.append(img_path)
+                data_labels.append(img_label)
+                data_encodings.append(self._generate_encoding(img_path))
         self.encoded_training_data['paths'] = data_paths
         self.encoded_training_data['labels'] = data_labels
         self.encoded_training_data['encodings'] = np.squeeze(
             np.array(data_encodings))
-        os.makedirs('encodings/', exist_ok=True)
-        f = open(os.path.join(encodings_path, save_file_name), "wb")
+
+        f = open(os.path.join(self.encodings_path, save_file_name), "wb")
         pickle.dump(self.encoded_training_data, f)
         f.close()
 
@@ -262,17 +252,18 @@ def load_encodings(self, path_to_encodings):
             print("Problem with encodings file")
 
     def calculate_distances(self, encoding):
-        dist = (
-            self.encoded_training_data['encodings'] - np.array(encoding))**2
-        dist = np.sum(dist, axis=1)
-        dist = np.sqrt(dist)
-        return dist
+        training_encodings = self.encoded_training_data['encodings']
+        return np.sqrt(
+            np.sum((training_encodings - np.array(encoding))**2, axis=1))
 
     def predict(self, image_path):
         img = cv2.imread(image_path)
         img = cv2.resize(img, (self.input_shape[0], self.input_shape[1]))
         encoding = self.base_model.predict(np.expand_dims(img, axis=0))
         distances = self.calculate_distances(encoding)
-        max_element = np.argmax(distances)
+        max_element = np.argmin(distances)
         predicted_label = self.encoded_training_data['labels'][max_element]
         return predicted_label
+
+    def calculate_prediction_accuracy(self):
+        pass

test_net.py

@@ -1,11 +1,14 @@
+import os
 from model import SiameseNet
 from data_loader import SiameseImageLoader
 import matplotlib.pyplot as plt
 from keras import optimizers
 import albumentations as A
+from keras.callbacks import TensorBoard, LearningRateScheduler
+from keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint
 
 
-def plot_grapth(values, y_label, title):
+def plot_grapth(values, y_label, title, project_name):
     t = list(range(len(values)))
     fig, ax = plt.subplots()
     ax.plot(t, values)
@@ -14,48 +17,83 @@ def plot_grapth(values, y_label, title):
            title='{}'.format(title))
     ax.grid()
 
-    fig.savefig("plots/{}.png".format(y_label))
+    fig.savefig("plots/{}{}.png".format(project_name, y_label))
 
 
-dataset_path = '/home/rauf/plates_competition/dataset/to_train/'
-n_epochs = 20
-n_steps_per_epoch = 600
-batch_size = 16
+project_name = 'road_signs/'
+dataset_path = '/home/rauf/plates_competition/dataset/road_signs/road_signs_separated/'
+n_epochs = 1000
+n_steps_per_epoch = 200
+batch_size = 32
 val_steps = 100
+input_shape = (75, 75, 3)
 
-augmentations = A.Compose([
-    A.RandomBrightnessContrast(p=0.4),
-    A.RandomGamma(p=0.4),
-    A.HueSaturationValue(hue_shift_limit=20,
-                         sat_shift_limit=50, val_shift_limit=50, p=0.4),
-    A.CLAHE(p=0.4),
-    A.HorizontalFlip(p=0.5),
-    A.VerticalFlip(p=0.5),
-    A.Blur(blur_limit=5, p=0.3),
-    A.GaussNoise(var_limit=(100, 150), p=0.3),
-    A.CenterCrop(p=1, height=256, width=256)
-], p=1)
+# augmentations = A.Compose([
+#     A.RandomBrightnessContrast(p=0.4),
+#     A.RandomGamma(p=0.4),
+#     A.HueSaturationValue(hue_shift_limit=20,
+#                          sat_shift_limit=50, val_shift_limit=50, p=0.4),
+#     A.CLAHE(p=0.4),
+#     A.HorizontalFlip(p=0.5),
+#     A.VerticalFlip(p=0.5),
+#     A.Blur(blur_limit=5, p=0.3),
+#     A.GaussNoise(var_limit=(100, 150), p=0.3),
+#     A.CenterCrop(p=1, height=256, width=256)
+# ], p=0.5)
 
-loader = SiameseImageLoader(dataset_path, input_shape=(
-    256, 256, 3), augmentations=augmentations)
+augmentations = None
 
+loader = SiameseImageLoader(
+    dataset_path, input_shape=input_shape, augmentations=augmentations)
 
-optimizer = optimizers.Adam(lr=1e-5)
+optimizer = optimizers.Adam(lr=1e-4)
 # optimizer = optimizers.RMSprop(lr=1e-5)
-model = SiameseNet(input_shape=(256, 256, 3), backbone='resnet50', mode='l2',
-                   image_loader=loader, optimizer=optimizer)
+# model = SiameseNet(input_shape=(256, 256, 3), backbone='resnet50', mode='l2',
+#                    image_loader=loader, optimizer=optimizer)
 
-train_losses, train_accuracies, val_losses, val_accuracies = model.train(
-    steps_per_epoch=n_steps_per_epoch, val_steps=val_steps, epochs=n_epochs)
+model = SiameseNet(input_shape=input_shape, backbone='resnet50', backbone_weights='imagenet', mode='l2',
+                   image_loader=loader, optimizer=optimizer, project_name=project_name,
+                   freeze_backbone=True)
 
-plot_grapth(train_losses, 'train_loss', 'Losses on train')
-plot_grapth(train_accuracies, 'train_acc', 'Accuracies on train')
-plot_grapth(val_losses, 'val_loss', 'Losses on val')
-plot_grapth(val_accuracies, 'val_acc', 'Accuracies on val')
+
+def step_decay_schedule(initial_lr=1e-3, decay_factor=0.75, step_size=10):
+    '''
+    Wrapper function to create a LearningRateScheduler with step decay schedule.
+    '''
+    def schedule(epoch):
+        return initial_lr * (decay_factor ** np.floor(epoch/step_size))
+
+    return LearningRateScheduler(schedule)
+
+
+callbacks = [
+    step_decay_schedule(initial_lr=1e-4, decay_factor=0.99, step_size=1),
+    EarlyStopping(patience=50, verbose=1),
+    TensorBoard(log_dir=SiameseNet.tensorboard_log_path),
+    # ReduceLROnPlateau(factor=0.9, patience=50,
+    #                   min_lr=1e-12, verbose=1),
+    ModelCheckpoint(filepath=os.path.join(SiameseNet.weights_save_path, 'best_model.hd5'), verbose=1, monitor='loss',
+                    save_best_only=True)
+]
+
+# train_losses, train_accuracies, val_losses, val_accuracies = model.train(
+#     steps_per_epoch=n_steps_per_epoch, val_steps=val_steps, epochs=n_epochs)
+
+H = model.train_generator(steps_per_epoch=n_steps_per_epoch, callbacks=callbacks,
+                          val_steps=val_steps, epochs=n_epochs)
+train_losses = H.history['loss']
+train_accuracies = H.history['accuracy']
+val_losses = H.history['val_loss']
+val_accuracies = H.history['val_accuracy']
+
+plot_grapth(train_losses, 'train_loss', 'Losses on train', project_name)
+plot_grapth(train_accuracies, 'train_acc', 'Accuracies on train', project_name)
+plot_grapth(val_losses, 'val_loss', 'Losses on val', project_name)
+plot_grapth(val_accuracies, 'val_acc', 'Accuracies on val', project_name)
 
 
 model.generate_encodings()
 # model.load_encodings('encodings/encodings.pkl')
 prediction = model.predict(
-    '/home/rauf/plates_competition/dataset/test/0000.jpg')
+    '/home/rauf/plates_competition/dataset/road_signs/road_signs_separated/val/7_1/rtsd-r3_test_009188.png')
 print(prediction)