-
Notifications
You must be signed in to change notification settings - Fork 0
/
iwp_index_wikipedia_34.py
70 lines (49 loc) · 2.22 KB
/
iwp_index_wikipedia_34.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
#******************************************************************************
# iwp_index_wikipedia_34.py
#******************************************************************************
from whoosh.index import *
from whoosh.fields import *
from whoosh import scoring
import xml.etree.ElementTree as ET
#------------------------------------------------------------------------------
#
# Create an initial index from files in a directory.
def iwp_create_index():
schema = Schema( title=TEXT( stored=True ),
path=ID( stored=True ), content=TEXT )
ix = create_in( "indexdir2", schema )
writer = ix.writer()
import glob
files = glob.glob( 'c:\\wikipedia\\resources\\wikipedia\\processed\\*\\*.xml' )
for file in files:
if file.find( 'part-60000\\1166530' ) == -1:
tree = ET.parse( file )
doc_elem = tree.getroot()
doc_text = ''.join( doc_elem.itertext() )
print( file )
writer.add_document( path=file, content=doc_text )
writer.commit()
#------------------------------------------------------------------------------
#
# Query an existing index using standard BM25 OKAPI weighting function.
def iwp_query_index_bm25( query ):
ix = open_dir("indexdir2")
from whoosh.qparser import QueryParser
with ix.searcher() as searcher:
query = QueryParser("content", ix.schema).parse( query )
results = searcher.search( query, limit=20 )
print( len( results ) )
for result in range( 0, results.scored_length() ):
print( results[ result ] )
#------------------------------------------------------------------------------
#
# Query an existing index using TF_IDF weighting function.
def iwp_query_index_tf_idf( query ):
ix = open_dir("indexdir2")
from whoosh.qparser import QueryParser
with ix.searcher( weighting=scoring.TF_IDF() ) as searcher:
query = QueryParser("content", ix.schema).parse( query )
results = searcher.search( query, limit=20 )
print( len( results ) )
for result in range( 0, results.scored_length() ):
print( results[ result ] )