add triplets generation

data_loader.py

@@ -35,22 +35,22 @@ def _load_images_paths(self):
                         self.images_paths[d].append(root+'/'+f)
                         self.images_labels[d].append(root.split('/')[-1])
 
-    def _get_pair(self, cl1, cl2, idx1, idx2, s='train', with_aug=True):
-        indx_1 = self.indexes[s][cl1][idx1]
-        indx_2 = self.indexes[s][cl2][idx2]
-        img_1 = cv2.imread(self.images_paths[s][indx_1])
-        img_2 = cv2.imread(self.images_paths[s][indx_2])
+
+    def _get_images_set(self, clsss, idxs, s='train', with_aug=True):
+
+        indxs = [self.indexes[s][cl][idx] for cl, idx in zip(clsss, idxs)]
+        imgs = [cv2.imread(self.images_paths[s][idx]) for idx in indxs]
+
         if self.input_shape:
-            img_1 = cv2.resize(
-                img_1, (self.input_shape[0], self.input_shape[1]))
-            img_2 = cv2.resize(
-                img_2, (self.input_shape[0], self.input_shape[1]))
+            imgs = [cv2.resize(
+                img, (self.input_shape[0], self.input_shape[1])) for img in imgs]
+
         if with_aug:
-            img_1 = self.augmentations(image=img_1)['image']
-            img_2 = self.augmentations(image=img_2)['image']
-        return img_1, img_2
+            imgs = [self.augmentations(image=img)['image'] for img in imgs]
+
+        return imgs
 
-    def get_batch(self, batch_size, s='train'):
+    def get_batch_pairs(self, batch_size,  s='train'):
         pairs = [np.zeros((batch_size, self.input_shape[0], self.input_shape[1], 3)), np.zeros(
             (batch_size, self.input_shape[0], self.input_shape[1], 3))]
         targets = np.zeros((batch_size,))
@@ -70,10 +70,10 @@ def get_batch(self, batch_size, s='train'):
             idx1 = indxs[i]
             idx2 = (idx1 + random.randrange(1, selected_class_n_elements)
                     ) % selected_class_n_elements
-            img1, img2 = self._get_pair(
-                selected_class, selected_class, idx1, idx2, s=s, with_aug=with_aug)
-            pairs[0][count, :, :, :] = img1
-            pairs[1][count, :, :, :] = img2
+            imgs = self._get_images_set(
+                [selected_class, selected_class], [idx1, idx2], s=s, with_aug=with_aug)
+            pairs[0][count, :, :, :] = imgs[0]
+            pairs[1][count, :, :, :] = imgs[1]
             targets[i] = 1
             count += 1
 
@@ -84,19 +84,58 @@ def get_batch(self, batch_size, s='train'):
             another_class_n_elements = len(self.indexes[s][another_class])
             idx1 = indxs[i]
             idx2 = random.randrange(0, another_class_n_elements)
-            img1, img2 = self._get_pair(
-                selected_class, another_class, idx1, idx2, s=s, with_aug=with_aug)
-            pairs[0][count, :, :, :] = img1
-            pairs[1][count, :, :, :] = img2
+            imgs = self._get_images_set(
+                [selected_class, another_class], [idx1, idx2], s=s, with_aug=with_aug)
+            pairs[0][count, :, :, :] = imgs[0]
+            pairs[1][count, :, :, :] = imgs[1]
             targets[i] = 0
             count += 1
 
         return pairs, targets
 
-    def generate(self, batch_size, s="train"):
+    def get_batch_triplets(self, batch_size,  s='train'):
+        triplets = [np.zeros((batch_size, self.input_shape[0], self.input_shape[1], 3)), 
+                    np.zeros((batch_size, self.input_shape[0], self.input_shape[1], 3)), 
+                    np.zeros((batch_size, self.input_shape[0], self.input_shape[1], 3))]
+        targets = np.zeros((batch_size,))
+
+        count = 0
+
+        for i in range(batch_size):
+            selected_class_idx = random.randrange(0, self.n_classes)
+            selected_class = self.classes[selected_class_idx]
+            selected_class_n_elements = len(self.indexes[s][selected_class])
+            another_class_idx = (
+                selected_class_idx + random.randrange(1, self.n_classes)) % self.n_classes
+            another_class = self.classes[another_class_idx]
+            another_class_n_elements = len(self.indexes[s][another_class])
+
+            indxs = np.random.randint(
+                selected_class_n_elements, size=batch_size)
+
+            with_aug = s == 'train' and self.augmentations
+            idx1 = indxs[i]
+            idx2 = (idx1 + random.randrange(1, selected_class_n_elements)
+                    ) % selected_class_n_elements
+            idx3 = random.randrange(0, another_class_n_elements)
+            imgs = self._get_images_set(
+                [selected_class, selected_class, another_class], [idx1, idx2, idx3], s=s, with_aug=with_aug)
+
+            triplets[0][count, :, :, :] = imgs[0]
+            triplets[1][count, :, :, :] = imgs[1]
+            triplets[2][count, :, :, :] = imgs[2]
+            targets[i] = 1
+            count += 1
+
+        return triplets, targets
+
+    def generate(self, batch_size, mode='pair', s="train"):
         while True:
-            pairs, targets = self.get_batch(batch_size, s)
-            yield (pairs, targets)
+            if mode == 'pair':
+                data, targets = self.get_batch_pairs(batch_size, s)
+            if mode == 'triplet':
+                data, targets = self.get_batch_triplets(batch_size, s)
+            yield (data, targets)
 
     def get_image(self, img_path):
         img = cv2.imread(img_path)

model.py

@@ -6,12 +6,12 @@
 import pickle
 import cv2
 import random
-from keras.models import Model
+from keras.models import Model, load_model
 from keras import optimizers
 from keras.regularizers import l2
 from keras.utils import plot_model
 from keras.layers import Dense, Input, Lambda, Dropout, Flatten
-from keras.layers import Conv2D, MaxPool2D
+from keras.layers import Conv2D, MaxPool2D, BatchNormalization
 from classification_models import Classifiers
 
 
@@ -49,17 +49,6 @@ def __init__(self, input_shape, image_loader, mode='l1', backbone='resnet50',
             os.makedirs(self.plots_path, exist_ok=True)
         if self.tensorboard_log_path:
             os.makedirs(self.tensorboard_log_path, exist_ok=True)
-        self.tensorboard_callback = TensorBoard(
-            self.tensorboard_log_path) if tensorboard_log_path else None
-        if self.tensorboard_callback:
-            events_files_list = glob.glob(
-                os.path.join(self.tensorboard_log_path, 'events*'))
-            for event_file in events_files_list:
-                try:
-                    os.remove(event_file)
-                except:
-                    print("Error while deleting file : ", event_file)
-            self.tensorboard_callback.set_model(self.model)
         self.weights_save_path = os.path.join(
             weights_save_path, self.project_name)
         if self.weights_save_path:
@@ -92,6 +81,40 @@ def _create_model(self):
                                    kernel_regularizer=l2(1e-3))(x)
             self.base_model = Model(
                 inputs=[input_image], outputs=[encoded_output])
+        elif self.backbone == 'simple2':
+            input_image = Input(self.input_shape)
+            x = Conv2D(32, kernel_size=3, activation='relu',
+                       kernel_regularizer=l2(2e-4))(input_image)
+            x = BatchNormalization()(x)
+            x = Conv2D(32, kernel_size=3, activation='relu',
+                       kernel_regularizer=l2(2e-4))(x)
+            x = BatchNormalization()(x)
+            x = Conv2D(32, kernel_size=5, strides=2, padding='same', activation='relu',
+                       kernel_regularizer=l2(2e-4))(x)
+            x = BatchNormalization()(x)
+            x = Dropout(0.4)(x)
+
+            x = Conv2D(64, kernel_size=3, activation='relu',
+                       kernel_regularizer=l2(2e-4))(x)
+            x = BatchNormalization()(x)
+            x = Conv2D(64, kernel_size=3, activation='relu',
+                       kernel_regularizer=l2(2e-4))(x)
+            x = BatchNormalization()(x)
+            x = Conv2D(64, kernel_size=5, strides=2, padding='same', activation='relu',
+                       kernel_regularizer=l2(2e-4))(x)
+            x = BatchNormalization()(x)
+            x = Dropout(0.4)(x)
+
+            x = Conv2D(128, kernel_size=4, activation='relu',
+                       kernel_regularizer=l2(2e-4))(x)
+            x = BatchNormalization()(x)
+            x = Flatten()(x)
+            x = Dense(512, activation="relu")(x)
+            x = Dropout(0.5)(x)
+            encoded_output = Dense(4096, activation='sigmoid',
+                                   kernel_regularizer=l2(1e-3))(x)
+            self.base_model = Model(
+                inputs=[input_image], outputs=[encoded_output])
         else:
             classifier, preprocess_input = Classifiers.get(self.backbone)
             backbone_model = classifier(
@@ -146,14 +169,6 @@ def _create_model(self):
         self.model.compile(loss=self.contrastive_loss, metrics=[metric],
                            optimizer=self.optimizer)
 
-    def write_log(self, names, logs, batch_no):
-        for name, value in zip(names, logs):
-            summary = tf.Summary()
-            summary_value = summary.value.add()
-            summary_value.simple_value = value
-            summary_value.tag = name
-            self.tensorboard_callback.writer.add_summary(summary, batch_no)
-            self.tensorboard_callback.writer.flush()
 
     def contrastive_loss(self, y_true, y_pred):
         '''Contrastive loss from Hadsell-et-al.'06
@@ -251,6 +266,13 @@ def load_encodings(self, path_to_encodings):
         except:
             print("Problem with encodings file")
 
+    def load_model(self,file_path):
+        self.model = load_model(file_path, 
+                                 custom_objects={'contrastive_loss': self.contrastive_loss, 
+                                                 'accuracy': self.accuracy})
+        self.base_model = Model(inputs=[self.model.layers[2].get_input_at(0)], 
+                                outputs=[self.model.layers[2].layers[-1].output])
+
     def calculate_distances(self, encoding):
         training_encodings = self.encoded_training_data['encodings']
         return np.sqrt(
@@ -259,11 +281,19 @@ def calculate_distances(self, encoding):
     def predict(self, image_path):
         img = cv2.imread(image_path)
         img = cv2.resize(img, (self.input_shape[0], self.input_shape[1]))
+        print(img.shape)
         encoding = self.base_model.predict(np.expand_dims(img, axis=0))
         distances = self.calculate_distances(encoding)
         max_element = np.argmin(distances)
         predicted_label = self.encoded_training_data['labels'][max_element]
         return predicted_label
 
     def calculate_prediction_accuracy(self):
-        pass
+        correct = 0
+        total_n_of_images = len(self.data_loader.images_paths['val'])
+        for img_path, img_label in zip(self.data_loader.images_paths['val'],
+                                       self.data_loader.images_labels['val']):
+            prediction = self.predict(img_path)
+            if prediction == img_label:
+                correct+=1
+        return correct/total_n_of_images

test_net.py

@@ -1,4 +1,5 @@
 import os
+import numpy as np
 from model import SiameseNet
 from data_loader import SiameseImageLoader
 import matplotlib.pyplot as plt
@@ -22,11 +23,15 @@ def plot_grapth(values, y_label, title, project_name):
 
 project_name = 'road_signs/'
 dataset_path = '/home/rauf/plates_competition/dataset/road_signs/road_signs_separated/'
+# project_name = 'plates/'
+# dataset_path = '/home/rauf/plates_competition/dataset/to_train/'
+
 n_epochs = 1000
-n_steps_per_epoch = 200
-batch_size = 32
+n_steps_per_epoch = 500
+batch_size = 4
 val_steps = 100
-input_shape = (75, 75, 3)
+input_shape = (48, 48, 3)
+# input_shape = (256, 256, 3)
 
 # augmentations = A.Compose([
 #     A.RandomBrightnessContrast(p=0.4),
@@ -51,28 +56,23 @@ def plot_grapth(values, y_label, title, project_name):
 # model = SiameseNet(input_shape=(256, 256, 3), backbone='resnet50', mode='l2',
 #                    image_loader=loader, optimizer=optimizer)
 
-model = SiameseNet(input_shape=input_shape, backbone='resnet50', backbone_weights='imagenet', mode='l2',
+model = SiameseNet(input_shape=input_shape, backbone='simple2', backbone_weights='imagenet', mode='l2',
                    image_loader=loader, optimizer=optimizer, project_name=project_name,
-                   freeze_backbone=True)
-
+                   freeze_backbone=False)
 
-def step_decay_schedule(initial_lr=1e-3, decay_factor=0.75, step_size=10):
-    '''
-    Wrapper function to create a LearningRateScheduler with step decay schedule.
-    '''
-    def schedule(epoch):
-        return initial_lr * (decay_factor ** np.floor(epoch/step_size))
-
-    return LearningRateScheduler(schedule)
 
+initial_lr = 1e-4
+decay_factor = 0.99
+step_size = 1
 
 callbacks = [
-    step_decay_schedule(initial_lr=1e-4, decay_factor=0.99, step_size=1),
-    EarlyStopping(patience=50, verbose=1),
-    TensorBoard(log_dir=SiameseNet.tensorboard_log_path),
+    LearningRateScheduler(lambda x: initial_lr *
+                          decay_factor ** np.floor(x/step_size)),
+    EarlyStopping(patience=100, verbose=1),
+    TensorBoard(log_dir=model.tensorboard_log_path),
     # ReduceLROnPlateau(factor=0.9, patience=50,
     #                   min_lr=1e-12, verbose=1),
-    ModelCheckpoint(filepath=os.path.join(SiameseNet.weights_save_path, 'best_model.hd5'), verbose=1, monitor='loss',
+    ModelCheckpoint(filepath=os.path.join(model.weights_save_path, 'best_model_2.h5'), verbose=1, monitor='loss',
                     save_best_only=True)
 ]
 
@@ -97,3 +97,6 @@ def schedule(epoch):
 prediction = model.predict(
     '/home/rauf/plates_competition/dataset/road_signs/road_signs_separated/val/7_1/rtsd-r3_test_009188.png')
 print(prediction)
+
+model_accuracy = model.calculate_prediction_accuracy()
+print('Model accuracy on validation set: {}'.format(model_accuracy))