second stage of algorithm selection

detection.txt

@@ -1,48 +1,18 @@
-248
-217
-333
-257
-73
-0.970254
-437
-284
-467
-300
-67
-0.788040
-20
-168
-137
-242
-24
-0.575427
-261
-147
-586
-462
-13
-0.860547
-172
-104
-455
-410
-13
-0.782301
-331
-82
-592
-483
-0
-0.998495
-37
-66
-267
-407
+130
+224
+170
+274
+58
+0.9600251
 0
-0.995558
-218
-85
-402
-459
-0
-0.988454
+242
+173
+374
+57
+0.95149636
+143
+255
+160
+274
+75
+0.7878658

main.py

@@ -20,6 +20,8 @@
 import random
 import itertools
 import colorsys
+from skimage.measure import regionprops
+import scipy.spatial.distance as dist
 
 app = Flask(__name__)
 
@@ -39,6 +41,27 @@
                'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors',
                'teddy bear', 'hair drier', 'toothbrush']
 
+catNames = ['person', 'bicycle', 'car', 'motorcycle', 'airplane',
+               'bus', 'train', 'truck', 'boat', 'traffic_light',
+               'fire_hydrant', 'stop_sign', 'parking_meter', 'bench', 'bird',
+               'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear',
+               'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie',
+               'suitcase', 'frisbee', 'skis', 'snowboard', 'sports_ball',
+               'kite', 'baseball_bat', 'baseball_glove', 'skateboard',
+               'surfboard', 'tennis_racket', 'bottle', 'wine_glass', 'cup',
+               'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple',
+               'sandwich', 'orange', 'broccoli', 'carrot', 'hot_dog', 'pizza',
+               'donut', 'cake', 'chair', 'couch', 'potted_plant', 'bed',
+               'dining_table', 'toilet', 'tv', 'laptop', 'mouse', 'remote',
+               'keyboard', 'cell_phone', 'microwave', 'oven', 'toaster',
+               'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors',
+               'teddy_bear', 'hair_drier', 'toothbrush']
+
+catIds = [1,2,3,4,5,6,7,8,9,10,11,13,14,15,16,17,18,19,20,21,22,23,24,25,
+             27,28,31,32,33,34,35,36,37,38,39,40,41,42,43,44,46,47,48,49,
+             50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,67,70,72,73,
+             74,75,76,77,78,79,80,81,82,84,85,86,87,88,89,90]
+
 def random_colors(N, bright=True):
     brightness = 1.0 if bright else 0.7
     hsv = [(i / N, 1, brightness) for i in range(N)]
@@ -51,11 +74,13 @@ def random_colors(N, bright=True):
 plt.rcParams['image.cmap'] = 'gray'
 
 algorithms = ['RetinaNet', 'SSD', 'MaskR-CNN', 'MultiPath Network', 'R-FCN', 'YOLO']
+cat_dir = '/mnt/images/MSCOCO/result_val2014/object_histograms/' # PATH TO OBJECT HISTOGRAMS
 
 UPLOAD_FOLDER = os.path.basename('uploads')
 app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
 
-clf = joblib.load('/mnt/images/MSCOCO/result_val2014/svc.sav') # PATH TO PRE-TRAINED MODEL
+wclf = joblib.load('/mnt/images/MSCOCO/result_val2014/svc.sav') # PATH TO PRE-TRAINED MODEL FOR WHOLE IMAGE
+iclf = joblib.load('/mnt/images/MSCOCO/result_val2014/dt.sav') # PATH TO PRE-TRAINED MODEL FOR INSTANCES
 model = models.alexnet(pretrained=True)
 new_classifier = nn.Sequential(*list(model.classifier.children())[:-1])
 
@@ -76,7 +101,7 @@ def __getitem__(self, index):
     def __len__(self):
         return len(self.__xs)
 
-def choose(fname, context):
+def chooseWhole(fname, context):
     batch_size = 1
     data_set = DataSet(fname)
     data_loader = torch.utils.data.DataLoader(data_set, batch_size=batch_size, shuffle=False, num_workers=1)
@@ -91,10 +116,63 @@ def choose(fname, context):
             outputs = outputs.view(batch_size, -1)
             for index, features in enumerate(outputs):
                 features = features.cpu().numpy()   
-    algo = clf.predict(np.hstack((features,context)).reshape(1, -1))[0]
+    algo = wclf.predict(np.hstack((features,context)).reshape(1, -1))[0]
+    return algo
+
+def norm(data):
+    data_norm = (data-min(data))/(max(data)-min(data))
+    weights = np.ones_like(data_norm)/float(len(data_norm))
+    hist = np.histogram(data_norm, weights=weights) 
+    return hist[0]
+
+def chooseInstance(fname, xmin, ymin, xmax, ymax, gt): 
+    img = Image.open(fname).convert('L')
+    width, height = img.size
+    obj = img.crop((xmin, ymin, xmax, ymax))
+    objWidth, objHeight = obj.size
+    regions = regionprops(obj)
+
+    obj_hist = []
+    data = sorted([r['area']/(width*height) for r in regions])[-50:]   
+    obj_hist = np.hstack((obj_hist, norm(data)))
+    centroidx = sorted([r['centroid'][0]/(width) for r in regions])[-50:]  
+    obj_hist = np.hstack((obj_hist, norm(data)))
+    centroidy = sorted([r['centroid'][1]/(height) for r in regions])[-50:]  
+    obj_hist = np.hstack((obj_hist, norm(data)))
+    eccentricities = sorted([r['eccentricity'] for r in regions])[-50:]  
+    obj_hist = np.hstack((obj_hist, norm(data)))
+    equivalent_diameters = sorted([r['equivalent_diameter']/r['area'] for r in regions])[-50:]   
+    obj_hist = np.hstack((obj_hist, norm(data)))
+    extents = sorted([r['extent'] for r in regions])[-50:]  
+    obj_hist = np.hstack((obj_hist, norm(data)))
+    orientations = sorted([r['orientation'] for r in regions])[-50:]  
+    obj_hist = np.hstack((obj_hist, norm(data)))
+    perimeters = sorted([r['perimeter']/((width+height)*2) for r in regions])[-50:]  
+    obj_hist = np.hstack((obj_hist, norm(data)))
+    solidities = sorted([r['solidity'] for r in regions])[-50:]  
+    obj_hist = np.hstack((obj_hist, norm(data)))
+
+    sims = np.empty((0,7))
+
+    for cl in catNames:
+        sim = []
+        obj = np.load('/mnt/images/MSCOCO/result_val2014/object_histograms/' + cl + '.npy')
+        sim.append(dist.braycurtis(obj_hist,obj))
+        sim.append(dist.canberra(obj_hist,obj))
+        sim.append(dist.cityblock(obj_hist,obj))
+        sim.append(dist.chebyshev(obj_hist,obj))
+        sim.append(dist.correlation(obj_hist,obj))
+        sim.append(dist.cosine(obj_hist,obj))
+        sim.append(dist.euclidean(obj_hist,obj))
+        sims = np.append(sims, np.expand_dims(sim, axis=0), axis=0)
+
+    gtsim = [gt]
+    for i in range(7):
+        gtsim.append(catIds[sims[:,i].tolist().index(min(sims[:,i]))])
+
+    algo = iclf.predict(gtsim)[0]
     return algo
 
-# ADD ALGORITHM AS PARAMETER
 def detect(fname, algo):
 
     if algo==1:
@@ -131,7 +209,6 @@ def detect(fname, algo):
         inst.append(category_id)
         inst.append(score)
         instances = np.append(instances, np.expand_dims(inst, axis=0), axis=0)    
-    # print(instances)
 
     colors = random_colors(instances.shape[0])
 
@@ -146,7 +223,7 @@ def detect(fname, algo):
         display_txt = class_names[(int(p[4]))] + ': ' + str(p[5])
         currentAxis.add_patch(plt.Rectangle(*coords, fill=False, edgecolor=color, linewidth=2))
         currentAxis.text(p[0], p[1], display_txt, bbox={'facecolor':color, 'alpha':0.5})
-      
+
     plt.savefig('detections/' + str(algorithms[algo-1]) + '_' + fname[30:])
     plt.close()
 
@@ -168,9 +245,10 @@ def upload_file():
     file = request.files['image'] 
     f = os.path.join(app.config['UPLOAD_FOLDER'], file.filename)
     file.save(f)
-    algo = choose('/home/alinoleumm/assv/uploads/' + str(file.filename), [place,inside,outside,filled,light,surrounding,time,action])
-    detect('/home/alinoleumm/assv/uploads/' + str(file.filename), algo) 
-    return 'Best algorithm for this image is ' + algorithms[algo-1] + '.'
+    algowhole = chooseWhole('/home/alinoleumm/assv/uploads/' + str(file.filename), [place,inside,outside,filled,light,surrounding,time,action])
+    detect('/home/alinoleumm/assv/uploads/' + str(file.filename), algo)
+    algoinst = chooseInstance('/home/alinoleumm/assv/uploads/' + str(file.filename), 0,242,173,374,57)
+    return 'Best algorithm for this image is ' + algorithms[algowhole-1] + ' and best algorithm for this instance is ' + algorithms[algoinst-1] + '.'
 
 if __name__ == "__main__":
     app.run(debug=True)