-
Notifications
You must be signed in to change notification settings - Fork 0
/
VQA.py
77 lines (74 loc) · 3.03 KB
/
VQA.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
from keras.models import model_from_json
from keras.preprocessing.sequence import pad_sequences
from nltk.tokenize import word_tokenize
from time import sleep
import numpy as np
import h5py
import json
from keras.applications.vgg19 import VGG19
from keras.applications.vgg19 import preprocess_input, decode_predictions
from keras.preprocessing import image
from keras.models import Model
# import redis
import pickle
import os
from tqdm import tqdm
PATH="resources"
base_model = VGG19(weights='imagenet')
model_vgg = Model(input=base_model.input, output=base_model.get_layer('fc2').output)
model_filename = os.path.join(PATH,'model.json')
model_weights_filename = os.path.join(PATH,'model_weights.h5')
metadata = json.load(open(os.path.join(PATH,'data_prepro.json'), 'r'))
metadata['ix_to_word'] = {str(word):int(i) for i,word in metadata['ix_to_word'].items()}
def load_image_array(image_file):
img = image.load_img(image_file, target_size=(224, 224))
img = image.img_to_array(img)
img = np.expand_dims(img, axis=0)
img = preprocess_input(img)
return img
def get_ques_vector(question):
question_vector = []
seq_length = 26
word_index = metadata['ix_to_word']
for word in word_tokenize(question.lower()):
if word in word_index:
question_vector.append(word_index[word])
else:
question_vector.append(0)
question_vector = np.array(pad_sequences([question_vector], maxlen=seq_length))[0]
question_vector = question_vector.reshape((1,seq_length))
return question_vector
def predict(image_path="kite.jpeg",question="What is flying in the sky?"):
json_file = open(model_filename, 'r')
loaded_model_json = json_file.read()
print "Reading Model..."
model = model_from_json(loaded_model_json)
print "Loading Weights..."
model.load_weights(model_weights_filename)
img = load_image_array(image_path)
img_vector = model_vgg.predict(img)
question_vector = get_ques_vector(question)
model.compile(optimizer='rmsprop', loss='categorical_crossentropy')
pred = model.predict([img_vector, question_vector])[0]
top_pred = pred.argsort()[-5:][::-1]
return [(metadata['ix_to_ans'][str(_)].title(), round(pred[_]*100.0,2)) for _ in top_pred]
# print (preds)
# predict()
def eval(annotations):
res = []
json_file = open(model_filename, 'r')
loaded_model_json = json_file.read()
print "Reading Model..."
model = model_from_json(loaded_model_json)
print "Loading Weights..."
model.load_weights(model_weights_filename)
model.compile(optimizer='rmsprop', loss='categorical_crossentropy')
for path in tqdm(annotations):
img = load_image_array(path["img_path"])
img_vector = model_vgg.predict(img)
question_vector = get_ques_vector(path[question])
pred = model.predict([img_vector, question_vector])[0]
top_pred = pred.argsort()[-5:][::-1]
res.append({"answer":metadata['ix_to_ans'][str(top_pred[0])].title(),"question_id":path["ques_id"]})
print "Result ready"
json.dump(res,open("result.json"))