fix problem with double entries in enrichment fields

sakshee-19 · Sep 8, 2018 · e36c442 · e36c442
1 parent c06e3d7
commit e36c442
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 5 deletions.
diff --git a/src/org/loklak/harvester/TwitterScraper.java b/src/org/loklak/harvester/TwitterScraper.java
@@ -28,7 +28,6 @@
 
 import java.net.MalformedURLException;
 import java.net.URL;
-import java.net.URLDecoder;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashMap;

diff --git a/src/org/loklak/objects/MessageEntry.java b/src/org/loklak/objects/MessageEntry.java
@@ -23,6 +23,7 @@
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.HashSet;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -72,16 +73,18 @@ public double getClassifierProbability(Classifier.Context context) {
 
     public static List<String> extract(String text, Pattern p, int regexGroup) {
         Matcher m = p.matcher(text);
-        List<String> dataList = new ArrayList<String>();
+        Set<String> dataList = new LinkedHashSet<String>();
 
         while (m.find()) {
             dataList.add(m.group(regexGroup));
         }
         for (String r: dataList) {
-            //text.replaceAll(r, "");
-            text.replace(r, "");
+            text.replace(r, ""); // replaceAll??
         }
-        return dataList;
+
+        List<String> result = new ArrayList<>(dataList.size());
+        result.addAll(dataList);
+        return result;
     }
 
    public List<String> extractLinks(String text) {