added refactoring

apps/target/classes/META-INF/services/zemberek.apps.ConsoleApp

@@ -0,0 +1,12 @@
+zemberek.apps.morphology.MorphologyConsole
+zemberek.apps.corpus.PreprocessTurkishCorpus
+zemberek.apps.fasttext.TrainClassifier
+zemberek.apps.fasttext.GenerateWordVectors
+zemberek.apps.fasttext.ClassificationConsole
+zemberek.apps.fasttext.EvaluateClassifier
+zemberek.apps.lm.CompressLm
+zemberek.apps.lm.GenerateVocabulary
+zemberek.apps.grpc.StartGrpcServer
+zemberek.apps.ner.EvaluateNer
+zemberek.apps.ner.TrainNerModel
+zemberek.apps.ner.FindNamedEntities

apps/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst

@@ -0,0 +1,19 @@
+zemberek\apps\corpus\RemoveDuplicateLines.class
+zemberek\apps\ConsoleApp.class
+zemberek\apps\ner\EvaluateNer.class
+zemberek\apps\corpus\PreprocessTurkishCorpus$Operation.class
+zemberek\apps\fasttext\GenerateWordVectors.class
+zemberek\apps\ner\TrainNerModel.class
+zemberek\apps\fasttext\EvaluateClassifier.class
+zemberek\apps\fasttext\ClassificationConsole$Preprocessor.class
+zemberek\apps\fasttext\FastTextAppBase.class
+zemberek\apps\fasttext\TrainClassifier.class
+zemberek\apps\lm\CompressLm.class
+zemberek\apps\ApplicationRunner.class
+zemberek\apps\lm\GenerateVocabulary.class
+zemberek\apps\ner\FindNamedEntities.class
+zemberek\apps\grpc\StartGrpcServer.class
+zemberek\apps\morphology\MorphologyConsole.class
+zemberek\apps\corpus\PreprocessTurkishCorpus.class
+zemberek\apps\ner\NerAppBase.class
+zemberek\apps\fasttext\ClassificationConsole.class

classification/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst

@@ -0,0 +1,4 @@
+zemberek\classification\FastTextClassifierTrainer.class
+zemberek\classification\FastTextClassifierTrainer$LossType.class
+zemberek\classification\FastTextClassifier.class
+zemberek\classification\FastTextClassifierTrainer$Builder.class

core/src/main/java/zemberek/core/collections/Histogram.java

@@ -25,6 +25,35 @@
  */
 public class Histogram<T> implements Iterable<T> {
 
+ public static class LoadHistogram{
+ public static Histogram<String> loadFromLines(
+ List<String> lines,
+ char delimiter,
+ boolean keyComesFirst) {
+ Histogram<String> result = new Histogram<>(lines.size());
+ for (String s : lines) {
+ int index = s.indexOf(delimiter);
+ if (index <= 0) {
+ throw new IllegalStateException("Bad histogram line = " + s);
+ }
+ String item = keyComesFirst ? s.substring(0, index) : s.substring(index + 1);
+ String countStr = keyComesFirst ? s.substring(index + 1) : s.substring(0, index);
+ int count = Integer.parseInt(countStr);
+ result.add(item, count);
+ }
+ return result;
+ }
+
+ public static Histogram<String> loadFromUtf8File(
+ Path path,
+ char delimiter,
+ boolean keyComesFirst) throws IOException {
+ List<String> lines = Files.readAllLines(path, StandardCharsets.UTF_8);
+ return loadFromLines(lines, delimiter, keyComesFirst);
+ }
+
+ }
+
  private final UIntValueMap<T> map;
 
  public Histogram(int initialSize) {
@@ -46,26 +75,27 @@ public Histogram() {
  * Loads from file with format: [key][delimiter][count]
  */
  public static Histogram<String> loadFromLines(List<String> lines, char delimiter) {
- return loadFromLines(lines, delimiter, true);
- }
-
- public static Histogram<String> loadFromLines(
- List<String> lines,
- char delimiter,
- boolean keyComesFirst) {
- Histogram<String> result = new Histogram<>(lines.size());
- for (String s : lines) {
- int index = s.indexOf(delimiter);
- if (index <= 0) {
- throw new IllegalStateException("Bad histogram line = " + s);
- }
- String item = keyComesFirst ? s.substring(0, index) : s.substring(index + 1);
- String countStr = keyComesFirst ? s.substring(index + 1) : s.substring(0, index);
- int count = Integer.parseInt(countStr);
- result.add(item, count);
- }
- return result;
- }
+ LoadHistogram histogram = new LoadHistogram();
+ return histogram.loadFromLines(lines, delimiter, true);
+ }
+
+// public static Histogram<String> loadFromLines(
+// List<String> lines,
+// char delimiter,
+// boolean keyComesFirst) {
+// Histogram<String> result = new Histogram<>(lines.size());
+// for (String s : lines) {
+// int index = s.indexOf(delimiter);
+// if (index <= 0) {
+// throw new IllegalStateException("Bad histogram line = " + s);
+// }
+// String item = keyComesFirst ? s.substring(0, index) : s.substring(index + 1);
+// String countStr = keyComesFirst ? s.substring(index + 1) : s.substring(0, index);
+// int count = Integer.parseInt(countStr);
+// result.add(item, count);
+// }
+// return result;
+// }
 
  /**
  * Loads a String Histogram from a file. Counts are supposedly delimited with `delimiter`
@@ -80,14 +110,13 @@ public static Histogram<String> loadFromUtf8File(Path path, char delimiter) thro
  return loadFromLines(lines, delimiter);
  }
 
- public static Histogram<String> loadFromUtf8File(
- Path path,
- char delimiter,
- boolean keyComesFirst) throws IOException {
- List<String> lines = Files.readAllLines(path, StandardCharsets.UTF_8);
- return loadFromLines(lines, delimiter, keyComesFirst);
- }
-
+// public static Histogram<String> loadFromUtf8File(
+// Path path,
+// char delimiter,
+// boolean keyComesFirst) throws IOException {
+// List<String> lines = Files.readAllLines(path, StandardCharsets.UTF_8);
+// return loadFromLines(lines, delimiter, keyComesFirst);
+// }
 
  public static void serializeStringHistogram(Histogram<String> h, DataOutputStream dos)
  throws IOException {
@@ -221,15 +250,7 @@ public void set(T t, int c) {
  }
 
  public int decrementIfPositive(T t) {
- if (t == null) {
- throw new NullPointerException("Element cannot be null");
- }
- int c = map.get(t);
- if (c > 0) {
- return map.decrement(t);
- } else {
- return 0;
- }
+ return map.decrementIfPositive(t);
  }
 
  /**
@@ -528,4 +549,4 @@ public int compare(Map.Entry<T, Integer> o1, Map.Entry<T, Integer> o2) {
  return (o2.getValue() < o1.getValue()) ? -1 : ((o2.getValue() > o1.getValue()) ? 1 : 0);
  }
  }
-}
+}

core/src/main/java/zemberek/core/collections/UIntValueMap.java

@@ -303,4 +303,16 @@ public void remove() {
  }
  }
 
+ public int decrementIfPositive(T t) {
+ if (t == null) {
+ throw new NullPointerException("Element cannot be null");
+ }
+ int c = this.get(t);
+ if (c > 0) {
+ return this.decrement(t);
+ } else {
+ return 0;
+ }
+ }
+
 }

core/target/classes/zemberek/core/syllable/accepted-syllable-prefixes

@@ -0,0 +1,62 @@
+bl
+br
+ch
+cl
+cr
+cy
+dj
+dr
+dz
+fl
+fr
+gh
+gl
+gr
+gy
+hr
+hy
+kh
+kl
+kn
+kr
+ks
+ky
+ll
+ly
+mb
+mc
+mn
+my
+ph
+pl
+pn
+pr
+ps
+pt
+rh
+sc
+sf
+sh
+sk
+sl
+sm
+sn
+sp
+sr
+st
+sv
+sw
+sy
+şl
+şn
+şv
+th
+tr
+ts
+tw
+ty
+vl
+wh
+zh
+zl
+zw