-
Notifications
You must be signed in to change notification settings - Fork 0
/
parsers.py
127 lines (106 loc) · 4.7 KB
/
parsers.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
from lxml import etree
from collections import namedtuple
from tqdm import tqdm, tnrange, tqdm_notebook
import json
import random
import utils
import numpy as np
class TrainingParser(object):
"""
Class to parse the XML training file
"""
def __init__(self, xml_file):
self.xml_file = xml_file
self.records = namedtuple("Training", "id_ lemma pos instance")
def count(self):
"""
Parses only for the sake of counting the number of sentences
return sum: of the sentences in the xml file
"""
return sum(1 for _ in etree.iterparse(self.xml_file, tag="sentence"))
def parse(self):
"""
Parses the archived training XML file in Raganato's format
:param archived_xml: path to XML file
:return: sentence generator
"""
for event, sentence in etree.iterparse(self.xml_file, tag="sentence"):
to_generator = []
if event == 'end':
for elem in sentence:
item = self.records(id_ = elem.get("id") if elem.tag == 'instance' else None,
lemma = elem.get("lemma"),
pos = elem.get("pos"),
instance = True if elem.tag == 'instance' else False)
to_generator.append(item)
yield to_generator
sentence.clear()
def create_vocab(self, input_vocab_path, pos_vocab_path, left_out_vocab_path, subsampling_rate=1e-3, min_count=5):
"""
Creates two vocabularies: Input and POS
:param input_vocab_path: path to save input_vocab
:param pos_vocab_path: path to save pos_vocab
:param left_out_vocab_path: path to save left out words
:param subsampling_rate: subsampling rate of the vocab http://mccormickml.com/2017/01/11/word2vec-tutorial-part-2-negative-sampling/
:param min_count: occurence of word
:return: None, JSON dumps to given paths
"""
input_vocab, pos_vocab = {}, {}
words_total, count = 0, 0
not_subsampled = set()
subsampling = True if min_count!=0 else False
for sentence in tqdm(self.parse()):
count += 1
for word in sentence:
words_total += 1
lemma, pos = word.lemma, word.pos
if word.instance == True:
not_subsampled.add(lemma)
# handling OOV
lemma, pos = utils.OOV_handeler(lemma, pos)
input_vocab[lemma] = input_vocab.get(lemma, 0) + 1
pos_vocab[pos] = pos_vocab.get(pos, 0) + 1
print("{} sentences parsed with {} words".format(count, len(input_vocab)))
#print(not_subsampled)
#print(not_subsampled)
#sort input vocab
input_vocab = dict(sorted(input_vocab.items(), key=lambda x: x[1], reverse=True))
#Subsampling
input_vocab_to_file = []
left_out_vocab = set()
for (word, occurence) in input_vocab.items():
if word in not_subsampled or occurence >= min_count:
prob = utils.probabilty_of_keeping_word(occurence, words_total, subsampling_rate, type_='mikolov2013') if subsampling else 0
if word in not_subsampled or random.uniform(0, 1) >= prob:
input_vocab_to_file.append(word)
else:
left_out_vocab.add(word)
print("subsampled input vocab: {}\nPOS vocab:{}\nleft out vocab:{}".format(len(input_vocab_to_file),
len(pos_vocab),
len(left_out_vocab)))
with open(left_out_vocab_path, 'w') as f:
json.dump(list(left_out_vocab), f)
with open(input_vocab_path, 'w') as f:
json.dump(input_vocab_to_file, f)
pos_vocab = sorted(pos_vocab.items(), key=lambda k: k[1], reverse=True)
pos_vocab = [i[0] for i in pos_vocab]
with open(pos_vocab_path, 'w') as f:
json.dump(pos_vocab, f)
class GoldParser(object):
"""
Class to parse the gold.txt file
"""
def __init__(self, file):
self.file = open(file, 'r')
self.records = namedtuple("Gold", "id_ senses")
def parse(self):
"""
Parses the archived gold text file
:return: sentence generator
"""
for line in self.file:
line = line.strip()
line = line.split(" ")
item = self.records(id_ = line[0],
senses = list(set(utils.WordNet.from_sensekey(i) for i in line[1:])))
yield item