-
Notifications
You must be signed in to change notification settings - Fork 3
/
train.py
134 lines (102 loc) · 3.57 KB
/
train.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
import numpy as np
import pandas as pd
from keras.models import load_model
from utils import load_data, cos_dis
from models import Combine_Categorical_Model, img_feature_model
inference = False
if not inference:
"""
Read Data
"""
data = load_data("data/0813")
X_train = data['X_train']
X_valid = data['X_valid']
Y_train = data['Y_train']
Y_valid = data['Y_valid']
tag2label = data['tag2label']
word_dict = data['word_dict']
"""
Pre-process
"""
# get label
train_word_input = [tag2label[tag2label['tag'] == x]['label'].tolist()[0] for x in Y_train['tag']]
valid_word_input = [tag2label[tag2label['tag'] == x]['label'].tolist()[0] for x in Y_valid['tag']]
# Embedding Labels
train_word_input = [word_dict[w] for w in train_word_input]
valid_word_input = [word_dict[w] for w in valid_word_input]
"""
Model Train
"""
# Y_train_list = list(map(lambda x: tag2int[x], Y_train['tag'].tolist()))
# Y_valid_list = list(map(lambda x: tag2int[x], Y_valid['tag'].tolist()))
#
# # one-hot encode tag types for loss calculation
# y_train = to_categorical(Y_train_list, num_classes=230)
# y_valid = to_categorical(Y_valid_list, num_classes=230)
#
# ccm = Combine_Categorical_Model()
# ccm.compile(optimizer='rmsprop', loss='categorical_crossentropy')
# history = ccm.fit([X_train, np.array(train_word_input)], y_train,
# validation_data=([X_valid, np.array(valid_word_input)], y_valid), epochs=20, verbose=1)
img2sematic = img_feature_model()
img2sematic.compile(optimizer='rmsprop', loss='MSE')
history = img2sematic.fit(X_train, np.array(train_word_input),
validation_data=(X_valid, np.array(valid_word_input)), epochs=20, verbose=1, batch_size=32)
img2sematic.save('img2sematic.h5')
keys = []
values = []
for key, value in word_dict.items():
keys.append(key)
values.append(value)
# Compare with every label, find the one with nearest cosine distance
# Valid
Pred_valid = img2sematic.predict(X_valid)
res_valid = []
for v1 in Pred_valid:
score = []
for v2 in values:
score.append(cos_dis(v1, v2))
label = keys[score.index(min(score))]
res_valid.append(tag2label[tag2label['label'] == label]['tag'].tolist())
pred_valid = [s[0] for s in res_valid]
ground_truth = Y_valid['tag'].tolist()
count = .0
for i in range(len(ground_truth)):
if ground_truth[i] == pred_valid[i]:
count += 1
print("Valid Accuracy:", round(count/len(ground_truth), 4))
else:
# Test Result
print("Start Inference...")
# Read Data
data = load_data("data/0813")
X_test = data['X_test']
test_files = data['test_files']
word_dict = data['word_dict']
tag2label = data['tag2label']
# Predict
model = load_model('img2sematic.h5')
Y_test = model.predict(X_test)
keys = []
values = []
for key, value in word_dict.items():
keys.append(key)
values.append(value)
res = []
for v1 in Y_test:
score = []
for v2 in values:
score.append(cos_dis(v1, v2))
label = keys[score.index(min(score))]
res.append(tag2label[tag2label['label'] == label]['tag'].tolist())
result = pd.concat([test_files, pd.DataFrame(res)], axis=1)
result.to_csv('submit.txt', sep='\t', header=None, index=None)
"""
Result Analysis
"""
zero_shot_tags = data['zero_shot_tags']
count = 0
for s in res:
if s[0] in zero_shot_tags:
count+=1
print('Prediction contains {} zero-shot tags'.format(count))