include stopword list, fix issues with stopword filter

buda-base · Jan 19, 2018 · 6edb146 · 6edb146
1 parent 3e548b0
commit 6edb146
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 13 deletions.
diff --git a/pom.xml b/pom.xml
@@ -149,6 +149,12 @@
           <include>total_lexicon.txt</include>
         </includes>
       </resource>
+      <resource>
+        <directory>src/main/resources/</directory>
+        <includes>
+          <include>*</include>
+        </includes>
+      </resource>
     </resources>
     <plugins>
        <plugin>

diff --git a/src/main/java/io/bdrc/lucene/bo/TibetanAnalyzer.java b/src/main/java/io/bdrc/lucene/bo/TibetanAnalyzer.java
@@ -32,7 +32,7 @@
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.CharArraySet;
 import org.apache.lucene.analysis.StopFilter;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.util.IOUtils;
 
@@ -76,7 +76,7 @@ public TibetanAnalyzer(boolean segmentInWords, boolean lemmatize, boolean filter
 		if (stopFilename != null) {
 			if (stopFilename.isEmpty()) {
 				InputStream stream = null;
-		        stream = TibetanAnalyzer.class.getResourceAsStream("/bo-stopwords.txt");
+		        stream = TibetanAnalyzer.class.getResourceAsStream("bo-stopwords.txt");
 		        if (stream == null) {      // we're not using the jar, there is no resource, assuming we're running the code
 		        	this.tibStopSet = null;
 		        } else {
@@ -149,29 +149,34 @@ protected Reader initReader(String fieldName, Reader reader) {
 	@Override
 	protected TokenStreamComponents createComponents(final String fieldName) {
 		Tokenizer source = null;
-		TokenStream filter = null;
+		TokenFilter filter = null;
 
 		if (segmentInWords) {
 			try {
 				source = new TibWordTokenizer();
-				if (lemmatize) {
-					((TibWordTokenizer) source).setLemmatize(lemmatize);
-				}
+				((TibWordTokenizer) source).setLemmatize(lemmatize);
 			} catch (FileNotFoundException e) {
 				e.printStackTrace();
 			} catch (IOException e) {
 				e.printStackTrace();
 			}
-			filter = new StopFilter(source, tibStopSet);
-
 		} else {
 			source = new TibSyllableTokenizer();
 			if (lemmatize) {
-				filter = (TibAffixedFilter) new TibAffixedFilter(source);
+				filter = new TibAffixedFilter(source);
 			}
-			filter = new StopFilter(filter, tibStopSet);
-		}		
-
-		return new TokenStreamComponents(source, filter);
+		}
+		if (tibStopSet != null) {
+			if (filter != null) {
+				filter = new StopFilter(filter, tibStopSet);
+			} else {
+				filter = new StopFilter(source, tibStopSet);
+			}
+		}
+		if (filter != null) {
+			return new TokenStreamComponents(source, filter);
+		} else {
+			return new TokenStreamComponents(source);
+		}
 	}
 }