-
Notifications
You must be signed in to change notification settings - Fork 0
/
txtutils.py
38 lines (29 loc) · 761 Bytes
/
txtutils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
import subprocess
import os
"""
TOKENIZED_FILE_PATH = 'tokens_obeliks.tsv'
def remove_tokens_file():
if os.path.exists(TOKENIZED_FILE_PATH):
os.remove(TOKENIZED_FILE_PATH)
"""
def tokenize(input_path, obeliks_path, output_path):
subprocess.run([
'java',
'-cp', obeliks_path+'target/classes',
'org.obeliks.Tokenizer',
'-if', input_path,
'-o', output_path,
'-tei'
])
"""
def get_sentences():
sentence = []
with open(TOKENIZED_FILE_PATH) as tokens:
for line in tokens:
parts = line.strip().split('\t')
if len(parts) == 2:
sentence.append(parts[1])
else:
yield sentence
sentence = []
"""