diff --git a/data_loader.py b/data_loader.py index 5e325e3..623d665 100644 --- a/data_loader.py +++ b/data_loader.py @@ -35,22 +35,22 @@ def _load_images_paths(self): self.images_paths[d].append(root+'/'+f) self.images_labels[d].append(root.split('/')[-1]) - def _get_pair(self, cl1, cl2, idx1, idx2, s='train', with_aug=True): - indx_1 = self.indexes[s][cl1][idx1] - indx_2 = self.indexes[s][cl2][idx2] - img_1 = cv2.imread(self.images_paths[s][indx_1]) - img_2 = cv2.imread(self.images_paths[s][indx_2]) + + def _get_images_set(self, clsss, idxs, s='train', with_aug=True): + + indxs = [self.indexes[s][cl][idx] for cl, idx in zip(clsss, idxs)] + imgs = [cv2.imread(self.images_paths[s][idx]) for idx in indxs] + if self.input_shape: - img_1 = cv2.resize( - img_1, (self.input_shape[0], self.input_shape[1])) - img_2 = cv2.resize( - img_2, (self.input_shape[0], self.input_shape[1])) + imgs = [cv2.resize( + img, (self.input_shape[0], self.input_shape[1])) for img in imgs] + if with_aug: - img_1 = self.augmentations(image=img_1)['image'] - img_2 = self.augmentations(image=img_2)['image'] - return img_1, img_2 + imgs = [self.augmentations(image=img)['image'] for img in imgs] + + return imgs - def get_batch(self, batch_size, s='train'): + def get_batch_pairs(self, batch_size, s='train'): pairs = [np.zeros((batch_size, self.input_shape[0], self.input_shape[1], 3)), np.zeros( (batch_size, self.input_shape[0], self.input_shape[1], 3))] targets = np.zeros((batch_size,)) @@ -70,10 +70,10 @@ def get_batch(self, batch_size, s='train'): idx1 = indxs[i] idx2 = (idx1 + random.randrange(1, selected_class_n_elements) ) % selected_class_n_elements - img1, img2 = self._get_pair( - selected_class, selected_class, idx1, idx2, s=s, with_aug=with_aug) - pairs[0][count, :, :, :] = img1 - pairs[1][count, :, :, :] = img2 + imgs = self._get_images_set( + [selected_class, selected_class], [idx1, idx2], s=s, with_aug=with_aug) + pairs[0][count, :, :, :] = imgs[0] + pairs[1][count, :, :, :] = imgs[1] targets[i] = 1 count += 1 @@ -84,19 +84,58 @@ def get_batch(self, batch_size, s='train'): another_class_n_elements = len(self.indexes[s][another_class]) idx1 = indxs[i] idx2 = random.randrange(0, another_class_n_elements) - img1, img2 = self._get_pair( - selected_class, another_class, idx1, idx2, s=s, with_aug=with_aug) - pairs[0][count, :, :, :] = img1 - pairs[1][count, :, :, :] = img2 + imgs = self._get_images_set( + [selected_class, another_class], [idx1, idx2], s=s, with_aug=with_aug) + pairs[0][count, :, :, :] = imgs[0] + pairs[1][count, :, :, :] = imgs[1] targets[i] = 0 count += 1 return pairs, targets - def generate(self, batch_size, s="train"): + def get_batch_triplets(self, batch_size, s='train'): + triplets = [np.zeros((batch_size, self.input_shape[0], self.input_shape[1], 3)), + np.zeros((batch_size, self.input_shape[0], self.input_shape[1], 3)), + np.zeros((batch_size, self.input_shape[0], self.input_shape[1], 3))] + targets = np.zeros((batch_size,)) + + count = 0 + + for i in range(batch_size): + selected_class_idx = random.randrange(0, self.n_classes) + selected_class = self.classes[selected_class_idx] + selected_class_n_elements = len(self.indexes[s][selected_class]) + another_class_idx = ( + selected_class_idx + random.randrange(1, self.n_classes)) % self.n_classes + another_class = self.classes[another_class_idx] + another_class_n_elements = len(self.indexes[s][another_class]) + + indxs = np.random.randint( + selected_class_n_elements, size=batch_size) + + with_aug = s == 'train' and self.augmentations + idx1 = indxs[i] + idx2 = (idx1 + random.randrange(1, selected_class_n_elements) + ) % selected_class_n_elements + idx3 = random.randrange(0, another_class_n_elements) + imgs = self._get_images_set( + [selected_class, selected_class, another_class], [idx1, idx2, idx3], s=s, with_aug=with_aug) + + triplets[0][count, :, :, :] = imgs[0] + triplets[1][count, :, :, :] = imgs[1] + triplets[2][count, :, :, :] = imgs[2] + targets[i] = 1 + count += 1 + + return triplets, targets + + def generate(self, batch_size, mode='pair', s="train"): while True: - pairs, targets = self.get_batch(batch_size, s) - yield (pairs, targets) + if mode == 'pair': + data, targets = self.get_batch_pairs(batch_size, s) + if mode == 'triplet': + data, targets = self.get_batch_triplets(batch_size, s) + yield (data, targets) def get_image(self, img_path): img = cv2.imread(img_path) diff --git a/model.py b/model.py index 990a20b..90b132a 100644 --- a/model.py +++ b/model.py @@ -6,12 +6,12 @@ import pickle import cv2 import random -from keras.models import Model +from keras.models import Model, load_model from keras import optimizers from keras.regularizers import l2 from keras.utils import plot_model from keras.layers import Dense, Input, Lambda, Dropout, Flatten -from keras.layers import Conv2D, MaxPool2D +from keras.layers import Conv2D, MaxPool2D, BatchNormalization from classification_models import Classifiers @@ -49,17 +49,6 @@ def __init__(self, input_shape, image_loader, mode='l1', backbone='resnet50', os.makedirs(self.plots_path, exist_ok=True) if self.tensorboard_log_path: os.makedirs(self.tensorboard_log_path, exist_ok=True) - self.tensorboard_callback = TensorBoard( - self.tensorboard_log_path) if tensorboard_log_path else None - if self.tensorboard_callback: - events_files_list = glob.glob( - os.path.join(self.tensorboard_log_path, 'events*')) - for event_file in events_files_list: - try: - os.remove(event_file) - except: - print("Error while deleting file : ", event_file) - self.tensorboard_callback.set_model(self.model) self.weights_save_path = os.path.join( weights_save_path, self.project_name) if self.weights_save_path: @@ -92,6 +81,40 @@ def _create_model(self): kernel_regularizer=l2(1e-3))(x) self.base_model = Model( inputs=[input_image], outputs=[encoded_output]) + elif self.backbone == 'simple2': + input_image = Input(self.input_shape) + x = Conv2D(32, kernel_size=3, activation='relu', + kernel_regularizer=l2(2e-4))(input_image) + x = BatchNormalization()(x) + x = Conv2D(32, kernel_size=3, activation='relu', + kernel_regularizer=l2(2e-4))(x) + x = BatchNormalization()(x) + x = Conv2D(32, kernel_size=5, strides=2, padding='same', activation='relu', + kernel_regularizer=l2(2e-4))(x) + x = BatchNormalization()(x) + x = Dropout(0.4)(x) + + x = Conv2D(64, kernel_size=3, activation='relu', + kernel_regularizer=l2(2e-4))(x) + x = BatchNormalization()(x) + x = Conv2D(64, kernel_size=3, activation='relu', + kernel_regularizer=l2(2e-4))(x) + x = BatchNormalization()(x) + x = Conv2D(64, kernel_size=5, strides=2, padding='same', activation='relu', + kernel_regularizer=l2(2e-4))(x) + x = BatchNormalization()(x) + x = Dropout(0.4)(x) + + x = Conv2D(128, kernel_size=4, activation='relu', + kernel_regularizer=l2(2e-4))(x) + x = BatchNormalization()(x) + x = Flatten()(x) + x = Dense(512, activation="relu")(x) + x = Dropout(0.5)(x) + encoded_output = Dense(4096, activation='sigmoid', + kernel_regularizer=l2(1e-3))(x) + self.base_model = Model( + inputs=[input_image], outputs=[encoded_output]) else: classifier, preprocess_input = Classifiers.get(self.backbone) backbone_model = classifier( @@ -146,14 +169,6 @@ def _create_model(self): self.model.compile(loss=self.contrastive_loss, metrics=[metric], optimizer=self.optimizer) - def write_log(self, names, logs, batch_no): - for name, value in zip(names, logs): - summary = tf.Summary() - summary_value = summary.value.add() - summary_value.simple_value = value - summary_value.tag = name - self.tensorboard_callback.writer.add_summary(summary, batch_no) - self.tensorboard_callback.writer.flush() def contrastive_loss(self, y_true, y_pred): '''Contrastive loss from Hadsell-et-al.'06 @@ -251,6 +266,13 @@ def load_encodings(self, path_to_encodings): except: print("Problem with encodings file") + def load_model(self,file_path): + self.model = load_model(file_path, + custom_objects={'contrastive_loss': self.contrastive_loss, + 'accuracy': self.accuracy}) + self.base_model = Model(inputs=[self.model.layers[2].get_input_at(0)], + outputs=[self.model.layers[2].layers[-1].output]) + def calculate_distances(self, encoding): training_encodings = self.encoded_training_data['encodings'] return np.sqrt( @@ -259,6 +281,7 @@ def calculate_distances(self, encoding): def predict(self, image_path): img = cv2.imread(image_path) img = cv2.resize(img, (self.input_shape[0], self.input_shape[1])) + print(img.shape) encoding = self.base_model.predict(np.expand_dims(img, axis=0)) distances = self.calculate_distances(encoding) max_element = np.argmin(distances) @@ -266,4 +289,11 @@ def predict(self, image_path): return predicted_label def calculate_prediction_accuracy(self): - pass \ No newline at end of file + correct = 0 + total_n_of_images = len(self.data_loader.images_paths['val']) + for img_path, img_label in zip(self.data_loader.images_paths['val'], + self.data_loader.images_labels['val']): + prediction = self.predict(img_path) + if prediction == img_label: + correct+=1 + return correct/total_n_of_images \ No newline at end of file diff --git a/test_net.py b/test_net.py index 06f6a30..1166e33 100644 --- a/test_net.py +++ b/test_net.py @@ -1,4 +1,5 @@ import os +import numpy as np from model import SiameseNet from data_loader import SiameseImageLoader import matplotlib.pyplot as plt @@ -22,11 +23,15 @@ def plot_grapth(values, y_label, title, project_name): project_name = 'road_signs/' dataset_path = '/home/rauf/plates_competition/dataset/road_signs/road_signs_separated/' +# project_name = 'plates/' +# dataset_path = '/home/rauf/plates_competition/dataset/to_train/' + n_epochs = 1000 -n_steps_per_epoch = 200 -batch_size = 32 +n_steps_per_epoch = 500 +batch_size = 4 val_steps = 100 -input_shape = (75, 75, 3) +input_shape = (48, 48, 3) +# input_shape = (256, 256, 3) # augmentations = A.Compose([ # A.RandomBrightnessContrast(p=0.4), @@ -51,28 +56,23 @@ def plot_grapth(values, y_label, title, project_name): # model = SiameseNet(input_shape=(256, 256, 3), backbone='resnet50', mode='l2', # image_loader=loader, optimizer=optimizer) -model = SiameseNet(input_shape=input_shape, backbone='resnet50', backbone_weights='imagenet', mode='l2', +model = SiameseNet(input_shape=input_shape, backbone='simple2', backbone_weights='imagenet', mode='l2', image_loader=loader, optimizer=optimizer, project_name=project_name, - freeze_backbone=True) - + freeze_backbone=False) -def step_decay_schedule(initial_lr=1e-3, decay_factor=0.75, step_size=10): - ''' - Wrapper function to create a LearningRateScheduler with step decay schedule. - ''' - def schedule(epoch): - return initial_lr * (decay_factor ** np.floor(epoch/step_size)) - - return LearningRateScheduler(schedule) +initial_lr = 1e-4 +decay_factor = 0.99 +step_size = 1 callbacks = [ - step_decay_schedule(initial_lr=1e-4, decay_factor=0.99, step_size=1), - EarlyStopping(patience=50, verbose=1), - TensorBoard(log_dir=SiameseNet.tensorboard_log_path), + LearningRateScheduler(lambda x: initial_lr * + decay_factor ** np.floor(x/step_size)), + EarlyStopping(patience=100, verbose=1), + TensorBoard(log_dir=model.tensorboard_log_path), # ReduceLROnPlateau(factor=0.9, patience=50, # min_lr=1e-12, verbose=1), - ModelCheckpoint(filepath=os.path.join(SiameseNet.weights_save_path, 'best_model.hd5'), verbose=1, monitor='loss', + ModelCheckpoint(filepath=os.path.join(model.weights_save_path, 'best_model_2.h5'), verbose=1, monitor='loss', save_best_only=True) ] @@ -97,3 +97,6 @@ def schedule(epoch): prediction = model.predict( '/home/rauf/plates_competition/dataset/road_signs/road_signs_separated/val/7_1/rtsd-r3_test_009188.png') print(prediction) + +model_accuracy = model.calculate_prediction_accuracy() +print('Model accuracy on validation set: {}'.format(model_accuracy))