From 9ff21cd50c42e34693429c8e646c079899e1eadf Mon Sep 17 00:00:00 2001 From: laura Date: Mon, 12 Jun 2023 16:37:49 +0200 Subject: [PATCH 01/29] changed structure of CoreNLPProvider: now possible to add CoreNLPProvider that sends requests to microservice --- .../informants/corenlp/CoreNLPProvider.java | 26 ++------------ .../informants/corenlp/ServiceAdapter.java | 5 +++ .../informants/corenlp/TextProcessor.java | 9 +++++ .../corenlp/TextProcessorFactory.java | 15 ++++++++ .../corenlp/TextProcessorLocal.java | 35 +++++++++++++++++++ .../corenlp/TextProcessorService.java | 11 ++++++ 6 files changed, 77 insertions(+), 24 deletions(-) create mode 100644 stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/ServiceAdapter.java create mode 100644 stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessor.java create mode 100644 stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorFactory.java create mode 100644 stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorLocal.java create mode 100644 stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/CoreNLPProvider.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/CoreNLPProvider.java index dd5542fde..1309096a3 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/CoreNLPProvider.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/CoreNLPProvider.java @@ -2,19 +2,14 @@ package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp; import java.util.Map; -import java.util.Properties; import edu.kit.kastel.mcse.ardoco.core.api.PreprocessingData; import edu.kit.kastel.mcse.ardoco.core.api.text.NlpInformant; import edu.kit.kastel.mcse.ardoco.core.api.text.Text; import edu.kit.kastel.mcse.ardoco.core.common.util.DataRepositoryHelper; import edu.kit.kastel.mcse.ardoco.core.data.DataRepository; -import edu.stanford.nlp.pipeline.CoreDocument; -import edu.stanford.nlp.pipeline.StanfordCoreNLP; public class CoreNLPProvider extends NlpInformant { - private static final String ANNOTATORS = "tokenize,ssplit,pos,parse,depparse,lemma"; // further: ",ner,coref" - private static final String DEPENDENCIES_ANNOTATION = "EnhancedPlusPlusDependenciesAnnotation"; private Text annotatedText; @@ -56,26 +51,9 @@ public synchronized Text getAnnotatedText() { return annotatedText; } - private static Properties getStanfordProperties(Properties properties) { - if (properties == null) { - throw new IllegalArgumentException("Properties are null"); - } - var allStanfordProperties = new Properties(properties); - allStanfordProperties.setProperty("annotators", ANNOTATORS); - - allStanfordProperties.put("parse", DEPENDENCIES_ANNOTATION); - allStanfordProperties.put("depparse", DEPENDENCIES_ANNOTATION); - allStanfordProperties.put("coref.algorithm", "fastneural"); - - return allStanfordProperties; - } - private Text processText(String inputText) { - Properties props = getStanfordProperties(new Properties()); - StanfordCoreNLP pipeline = new StanfordCoreNLP(props); - CoreDocument document = new CoreDocument(inputText); - pipeline.annotate(document); - return new TextImpl(document); + TextProcessor textProcessor = new TextProcessorFactory().createCoreNlpTextProcessor(); + return textProcessor.processText(inputText); } @Override diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/ServiceAdapter.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/ServiceAdapter.java new file mode 100644 index 000000000..eb0c6ad3f --- /dev/null +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/ServiceAdapter.java @@ -0,0 +1,5 @@ +package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp; + +public class ServiceAdapter { + // todo: implement +} diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessor.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessor.java new file mode 100644 index 000000000..cdb74de57 --- /dev/null +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessor.java @@ -0,0 +1,9 @@ +package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp; + +import edu.kit.kastel.mcse.ardoco.core.api.text.Text; + +public interface TextProcessor { + + Text processText(String inputText); + +} diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorFactory.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorFactory.java new file mode 100644 index 000000000..9612c5938 --- /dev/null +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorFactory.java @@ -0,0 +1,15 @@ +package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp; + +public class TextProcessorFactory { + + private String coreNlpProviderSrc = "Service"; // todo: config + + public TextProcessor createCoreNlpTextProcessor() { + if (coreNlpProviderSrc.equals("Service")) { + return new TextProcessorService(); + } else { + return new TextProcessorLocal(); + } + } + +} diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorLocal.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorLocal.java new file mode 100644 index 000000000..1d9ea8428 --- /dev/null +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorLocal.java @@ -0,0 +1,35 @@ +package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp; + +import edu.kit.kastel.mcse.ardoco.core.api.text.Text; +import edu.stanford.nlp.pipeline.CoreDocument; +import edu.stanford.nlp.pipeline.StanfordCoreNLP; + +import java.util.Properties; + +public class TextProcessorLocal implements TextProcessor { + private static final String ANNOTATORS = "tokenize,ssplit,pos,parse,depparse,lemma"; // further: ",ner,coref" + private static final String DEPENDENCIES_ANNOTATION = "EnhancedPlusPlusDependenciesAnnotation"; + + @Override + public Text processText(String inputText) { + Properties props = getStanfordProperties(new Properties()); + StanfordCoreNLP pipeline = new StanfordCoreNLP(props); + CoreDocument document = new CoreDocument(inputText); + pipeline.annotate(document); + return new TextImpl(document); + } + + private static Properties getStanfordProperties(Properties properties) { + if (properties == null) { + throw new IllegalArgumentException("Properties are null"); + } + var allStanfordProperties = new Properties(properties); + allStanfordProperties.setProperty("annotators", ANNOTATORS); + + allStanfordProperties.put("parse", DEPENDENCIES_ANNOTATION); + allStanfordProperties.put("depparse", DEPENDENCIES_ANNOTATION); + allStanfordProperties.put("coref.algorithm", "fastneural"); + + return allStanfordProperties; + } +} diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java new file mode 100644 index 000000000..437658b3b --- /dev/null +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java @@ -0,0 +1,11 @@ +package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp; + +import edu.kit.kastel.mcse.ardoco.core.api.text.Text; + +public class TextProcessorService implements TextProcessor { + + @Override + public Text processText(String inputText) { + return null; // todo: implement + } +} From c142648ef22800828fade3ea3ddb8907012ac22f Mon Sep 17 00:00:00 2001 From: laura Date: Mon, 12 Jun 2023 17:09:33 +0200 Subject: [PATCH 02/29] added textprovider-json dependency --- stages/text-preprocessing/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/stages/text-preprocessing/pom.xml b/stages/text-preprocessing/pom.xml index 88de55100..aa7c7fc54 100644 --- a/stages/text-preprocessing/pom.xml +++ b/stages/text-preprocessing/pom.xml @@ -57,5 +57,10 @@ slf4j-simple test + + io.github.ardoco + text-provider-json + 0.1.0 + From 775f172ee3d4cbd3246ae334504488cb51f3146b Mon Sep 17 00:00:00 2001 From: laura Date: Mon, 12 Jun 2023 17:31:03 +0200 Subject: [PATCH 03/29] implemented TextProcessorService --- .../informants/corenlp/ServiceAdapter.java | 5 --- .../corenlp/TextProcessorService.java | 40 ++++++++++++++++++- 2 files changed, 39 insertions(+), 6 deletions(-) delete mode 100644 stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/ServiceAdapter.java diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/ServiceAdapter.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/ServiceAdapter.java deleted file mode 100644 index eb0c6ad3f..000000000 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/ServiceAdapter.java +++ /dev/null @@ -1,5 +0,0 @@ -package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp; - -public class ServiceAdapter { - // todo: implement -} diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java index 437658b3b..6f766c24f 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java @@ -2,10 +2,48 @@ import edu.kit.kastel.mcse.ardoco.core.api.text.Text; +import io.github.ardoco.textproviderjson.converter.JsonConverter; +import io.github.ardoco.textproviderjson.converter.DtoToObjectConverter; +import io.github.ardoco.textproviderjson.dto.TextDTO; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; + public class TextProcessorService implements TextProcessor { + private final String microserviceAddress = "localhost:8080"; + private final String requestUrl = "http://" + microserviceAddress + "/stanfordnlp?text="; + @Override public Text processText(String inputText) { - return null; // todo: implement + TextDTO textDto; + try { + String jsonText = sendRequest(inputText); + textDto = JsonConverter.fromJsonString(jsonText); + } catch (IOException e) { + return null; // todo error handling + } + return new DtoToObjectConverter().convertText(textDto); + } + + private String sendRequest(String inputText) throws IOException { + inputText = inputText.replace(" ", "%20"); + URL url = new URL(requestUrl + inputText); + HttpURLConnection con = (HttpURLConnection) url.openConnection(); + con.setRequestMethod("GET"); +// int status = con.getResponseCode(); + BufferedReader in = new BufferedReader( + new InputStreamReader(con.getInputStream()) + ); + String inputLine; + StringBuilder content = new StringBuilder(); + while ((inputLine = in.readLine()) != null) { + content.append(inputLine); + } + in.close(); + return content.toString(); } } From 4a724f8f7ebe0f2d01114a29f10301283f70bfc5 Mon Sep 17 00:00:00 2001 From: laura Date: Tue, 13 Jun 2023 09:34:00 +0200 Subject: [PATCH 04/29] added config --- .../corenlp/microservice/ConfigManager.java | 33 +++++++++++++++++++ .../corenlp/microservice/config.properties | 1 + 2 files changed, 34 insertions(+) create mode 100644 stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/microservice/ConfigManager.java create mode 100644 stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/microservice/config.properties diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/microservice/ConfigManager.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/microservice/ConfigManager.java new file mode 100644 index 000000000..543bcd7d4 --- /dev/null +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/microservice/ConfigManager.java @@ -0,0 +1,33 @@ +package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.microservice; + +import java.io.FileInputStream; +import java.io.IOException; +import java.util.Properties; + +/** + * This Singleton has access to the config file. + */ +public class ConfigManager { + private static ConfigManager instance; + private Properties properties; + + private ConfigManager(String filePath) { + properties = new Properties(); + try (FileInputStream fileInputStream = new FileInputStream(filePath);) { + properties.load(fileInputStream); + } catch (IOException e) { + e.printStackTrace(); + } + } + + public static ConfigManager getInstance(String filePath) { + if (instance == null) { + instance = new ConfigManager(filePath); + } + return instance; + } + + public String getProperty(String key) { + return properties.getProperty(key); + } +} \ No newline at end of file diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/microservice/config.properties b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/microservice/config.properties new file mode 100644 index 000000000..b966b8229 --- /dev/null +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/microservice/config.properties @@ -0,0 +1 @@ +port=8080 \ No newline at end of file From f568d62ecb374e86664fed5183ed37876a9ca25d Mon Sep 17 00:00:00 2001 From: laura Date: Tue, 13 Jun 2023 09:40:44 +0200 Subject: [PATCH 05/29] refactored ConfigManager --- .../{microservice => config}/ConfigManager.java | 11 ++++++----- .../informants/corenlp/config/config.properties | 1 + .../informants/corenlp/microservice/config.properties | 1 - 3 files changed, 7 insertions(+), 6 deletions(-) rename stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/{microservice => config}/ConfigManager.java (74%) create mode 100644 stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/config.properties delete mode 100644 stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/microservice/config.properties diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/microservice/ConfigManager.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java similarity index 74% rename from stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/microservice/ConfigManager.java rename to stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java index 543bcd7d4..be540354b 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/microservice/ConfigManager.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java @@ -1,4 +1,4 @@ -package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.microservice; +package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config; import java.io.FileInputStream; import java.io.IOException; @@ -9,9 +9,10 @@ */ public class ConfigManager { private static ConfigManager instance; - private Properties properties; + private final Properties properties; + private final String filePath = "config.properties"; - private ConfigManager(String filePath) { + private ConfigManager() { properties = new Properties(); try (FileInputStream fileInputStream = new FileInputStream(filePath);) { properties.load(fileInputStream); @@ -20,9 +21,9 @@ private ConfigManager(String filePath) { } } - public static ConfigManager getInstance(String filePath) { + public static ConfigManager getInstance() { if (instance == null) { - instance = new ConfigManager(filePath); + instance = new ConfigManager(); } return instance; } diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/config.properties b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/config.properties new file mode 100644 index 000000000..c170d49dc --- /dev/null +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/config.properties @@ -0,0 +1 @@ +nlpProviderSource=microservice \ No newline at end of file diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/microservice/config.properties b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/microservice/config.properties deleted file mode 100644 index b966b8229..000000000 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/microservice/config.properties +++ /dev/null @@ -1 +0,0 @@ -port=8080 \ No newline at end of file From 6208ad9c7e423cdc77e0cad4e7557820a4fc3a36 Mon Sep 17 00:00:00 2001 From: laura Date: Tue, 13 Jun 2023 10:27:44 +0200 Subject: [PATCH 06/29] added health check for service --- .../corenlp/TextProcessorFactory.java | 7 ++-- .../corenlp/TextProcessorService.java | 35 +++++++++++++++---- .../corenlp/config/config.properties | 6 +++- 3 files changed, 38 insertions(+), 10 deletions(-) diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorFactory.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorFactory.java index 9612c5938..98bcdb08e 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorFactory.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorFactory.java @@ -1,11 +1,12 @@ package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp; -public class TextProcessorFactory { +import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config.ConfigManager; - private String coreNlpProviderSrc = "Service"; // todo: config +public class TextProcessorFactory { public TextProcessor createCoreNlpTextProcessor() { - if (coreNlpProviderSrc.equals("Service")) { + if (ConfigManager.getInstance().getProperty("nlpProviderSource").equals("microservice") + && TextProcessorService.isMicroserviceAvailable()) { return new TextProcessorService(); } else { return new TextProcessorLocal(); diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java index 6f766c24f..6c735a77f 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java @@ -2,6 +2,7 @@ import edu.kit.kastel.mcse.ardoco.core.api.text.Text; +import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config.ConfigManager; import io.github.ardoco.textproviderjson.converter.JsonConverter; import io.github.ardoco.textproviderjson.converter.DtoToObjectConverter; import io.github.ardoco.textproviderjson.dto.TextDTO; @@ -14,14 +15,11 @@ public class TextProcessorService implements TextProcessor { - private final String microserviceAddress = "localhost:8080"; - private final String requestUrl = "http://" + microserviceAddress + "/stanfordnlp?text="; - @Override public Text processText(String inputText) { TextDTO textDto; try { - String jsonText = sendRequest(inputText); + String jsonText = sendCorenlpRequest(inputText); textDto = JsonConverter.fromJsonString(jsonText); } catch (IOException e) { return null; // todo error handling @@ -29,9 +27,34 @@ public Text processText(String inputText) { return new DtoToObjectConverter().convertText(textDto); } - private String sendRequest(String inputText) throws IOException { + public static boolean isMicroserviceAvailable() { + String requestUrl = ConfigManager.getInstance().getProperty("microserviceUrl") + ConfigManager.getInstance().getProperty("healthService"); + try { + URL url = new URL(requestUrl); + HttpURLConnection con = (HttpURLConnection) url.openConnection(); + con.setRequestMethod("GET"); + con.setConnectTimeout(5000); // timeout after 5 sec + int statusCode = con.getResponseCode(); + con.disconnect(); + if (statusCode != 200) { + return false; + } + } catch (Exception e) { + return false; + } + return true; + } + + private String sendCorenlpRequest(String inputText) throws IOException { inputText = inputText.replace(" ", "%20"); - URL url = new URL(requestUrl + inputText); + String requestUrl = ConfigManager.getInstance().getProperty("microserviceUrl") + + ConfigManager.getInstance().getProperty("corenlpService") + + inputText; + return sendGetRequest(requestUrl); + } + + private String sendGetRequest(String requestUrl) throws IOException { + URL url = new URL(requestUrl); HttpURLConnection con = (HttpURLConnection) url.openConnection(); con.setRequestMethod("GET"); // int status = con.getResponseCode(); diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/config.properties b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/config.properties index c170d49dc..bfb9ab9cb 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/config.properties +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/config.properties @@ -1 +1,5 @@ -nlpProviderSource=microservice \ No newline at end of file +nlpProviderSource=microservice +microserviceUrl= http://localhost:8080 +corenlpService=/stanfordnlp?text= +healthService=/health + From fb8dd82240368d5db7b7b2e22a1dc6d4dc923888 Mon Sep 17 00:00:00 2001 From: laura Date: Tue, 13 Jun 2023 13:43:00 +0200 Subject: [PATCH 07/29] added url encoding and changed filepath of config file --- .../informants/corenlp/TextProcessorService.java | 8 ++++++-- .../informants/corenlp/config/ConfigManager.java | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java index 6c735a77f..42ce32122 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java @@ -12,6 +12,8 @@ import java.io.InputStreamReader; import java.net.HttpURLConnection; import java.net.URL; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; public class TextProcessorService implements TextProcessor { @@ -46,7 +48,7 @@ public static boolean isMicroserviceAvailable() { } private String sendCorenlpRequest(String inputText) throws IOException { - inputText = inputText.replace(" ", "%20"); + inputText = URLEncoder.encode(inputText, StandardCharsets.UTF_8); String requestUrl = ConfigManager.getInstance().getProperty("microserviceUrl") + ConfigManager.getInstance().getProperty("corenlpService") + inputText; @@ -57,7 +59,9 @@ private String sendGetRequest(String requestUrl) throws IOException { URL url = new URL(requestUrl); HttpURLConnection con = (HttpURLConnection) url.openConnection(); con.setRequestMethod("GET"); -// int status = con.getResponseCode(); + if (con.getResponseCode() != HttpURLConnection.HTTP_OK) { + return null; // TODO error handling + } BufferedReader in = new BufferedReader( new InputStreamReader(con.getInputStream()) ); diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java index be540354b..8acd350dd 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java @@ -10,7 +10,7 @@ public class ConfigManager { private static ConfigManager instance; private final Properties properties; - private final String filePath = "config.properties"; + private final String filePath = "src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/config.properties"; private ConfigManager() { properties = new Properties(); From 5045fd52470a5539b5f4f7fc1608253d24b46fb9 Mon Sep 17 00:00:00 2001 From: laura Date: Tue, 13 Jun 2023 14:58:04 +0200 Subject: [PATCH 08/29] added javadoc --- .../informants/corenlp/TextProcessor.java | 5 +++++ .../informants/corenlp/TextProcessorFactory.java | 1 + .../informants/corenlp/TextProcessorLocal.java | 3 +++ .../informants/corenlp/TextProcessorService.java | 9 ++++++++- .../informants/corenlp/config/ConfigManager.java | 16 +++++++++++++++- 5 files changed, 32 insertions(+), 2 deletions(-) diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessor.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessor.java index cdb74de57..8d0ae19da 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessor.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessor.java @@ -4,6 +4,11 @@ public interface TextProcessor { + /** + * processes and annotates a given text + * @param inputText the input text + * @return the annotated text + */ Text processText(String inputText); } diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorFactory.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorFactory.java index 98bcdb08e..21c03f4fc 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorFactory.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorFactory.java @@ -7,6 +7,7 @@ public class TextProcessorFactory { public TextProcessor createCoreNlpTextProcessor() { if (ConfigManager.getInstance().getProperty("nlpProviderSource").equals("microservice") && TextProcessorService.isMicroserviceAvailable()) { + // return a text processor that uses the CoreNLP microservice return new TextProcessorService(); } else { return new TextProcessorLocal(); diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorLocal.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorLocal.java index 1d9ea8428..9160560ce 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorLocal.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorLocal.java @@ -6,6 +6,9 @@ import java.util.Properties; +/** + * This text processor processes texts locally using CoreNLP. + */ public class TextProcessorLocal implements TextProcessor { private static final String ANNOTATORS = "tokenize,ssplit,pos,parse,depparse,lemma"; // further: ",ner,coref" private static final String DEPENDENCIES_ANNOTATION = "EnhancedPlusPlusDependenciesAnnotation"; diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java index 42ce32122..5b4999417 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java @@ -15,6 +15,9 @@ import java.net.URLEncoder; import java.nio.charset.StandardCharsets; +/** + * This text processor processes texts by sending requests to a microservice, which provides text processing using CoreNLP. + */ public class TextProcessorService implements TextProcessor { @Override @@ -29,6 +32,10 @@ public Text processText(String inputText) { return new DtoToObjectConverter().convertText(textDto); } + /** + * checks if the CoreNLP microservice is available and can provide its services. + * @return whether the microservice is available + */ public static boolean isMicroserviceAvailable() { String requestUrl = ConfigManager.getInstance().getProperty("microserviceUrl") + ConfigManager.getInstance().getProperty("healthService"); try { @@ -38,7 +45,7 @@ public static boolean isMicroserviceAvailable() { con.setConnectTimeout(5000); // timeout after 5 sec int statusCode = con.getResponseCode(); con.disconnect(); - if (statusCode != 200) { + if (statusCode != HttpURLConnection.HTTP_OK) { return false; } } catch (Exception e) { diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java index 8acd350dd..ce44ca164 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java @@ -5,7 +5,7 @@ import java.util.Properties; /** - * This Singleton has access to the config file. + * This Singleton manages access to the config file. */ public class ConfigManager { private static ConfigManager instance; @@ -28,7 +28,21 @@ public static ConfigManager getInstance() { return instance; } + /** + * gets the value of the given key in the config file + * @param key the key + * @return the value + */ public String getProperty(String key) { return properties.getProperty(key); } + + /** + * sets the value of the given key in the config file + * @param key the key + * @param value the new value + */ + public void setProperty(String key, String value) { + properties.setProperty(key, value); + } } \ No newline at end of file From 7c75e80d265abf7a142176a3ad9d271956217c5a Mon Sep 17 00:00:00 2001 From: laura Date: Sat, 24 Jun 2023 14:03:51 +0200 Subject: [PATCH 09/29] add MicroserviceChecker --- .../informants/corenlp/CoreNLPProvider.java | 1 + .../corenlp/MicroserviceChecker.java | 38 +++++++++++++++++++ .../informants/corenlp/TextImpl.java | 2 +- .../corenlp/TextProcessorFactory.java | 5 ++- .../{ => textprocessor}/TextProcessor.java | 2 +- .../TextProcessorLocal.java | 3 +- .../TextProcessorService.java | 24 +----------- 7 files changed, 48 insertions(+), 27 deletions(-) create mode 100644 stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/MicroserviceChecker.java rename stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/{ => textprocessor}/TextProcessor.java (93%) rename stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/{ => textprocessor}/TextProcessorLocal.java (93%) rename stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/{ => textprocessor}/TextProcessorService.java (72%) diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/CoreNLPProvider.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/CoreNLPProvider.java index 1309096a3..842e0c49e 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/CoreNLPProvider.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/CoreNLPProvider.java @@ -8,6 +8,7 @@ import edu.kit.kastel.mcse.ardoco.core.api.text.Text; import edu.kit.kastel.mcse.ardoco.core.common.util.DataRepositoryHelper; import edu.kit.kastel.mcse.ardoco.core.data.DataRepository; +import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor.TextProcessor; public class CoreNLPProvider extends NlpInformant { diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/MicroserviceChecker.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/MicroserviceChecker.java new file mode 100644 index 000000000..2c3c56fcf --- /dev/null +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/MicroserviceChecker.java @@ -0,0 +1,38 @@ +package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp; + +import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config.ConfigManager; + +import java.net.HttpURLConnection; +import java.net.URL; + +/** + * This utility class provides methods to check whether the microservice is available. + */ +public final class MicroserviceChecker { + + private MicroserviceChecker() { + } + + /** + * checks if the CoreNLP microservice is available and can provide its services. + * @return whether the microservice is available + */ + public static boolean isMicroserviceAvailable() { + String requestUrl = ConfigManager.getInstance().getProperty("microserviceUrl") + ConfigManager.getInstance().getProperty("healthService"); + try { + URL url = new URL(requestUrl); + HttpURLConnection con = (HttpURLConnection) url.openConnection(); + con.setRequestMethod("GET"); + con.setConnectTimeout(5000); // timeout after 5 sec + int statusCode = con.getResponseCode(); + con.disconnect(); + if (statusCode != HttpURLConnection.HTTP_OK) { + return false; + } + } catch (Exception e) { + return false; + } + return true; + } + +} diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextImpl.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextImpl.java index bed00c5bf..22b7d989a 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextImpl.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextImpl.java @@ -10,7 +10,7 @@ import edu.kit.kastel.mcse.ardoco.core.api.text.Word; import edu.stanford.nlp.pipeline.CoreDocument; -class TextImpl implements Text { +public class TextImpl implements Text { final CoreDocument coreDocument; private ImmutableList sentences = Lists.immutable.empty(); diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorFactory.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorFactory.java index 21c03f4fc..ed7f2d12e 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorFactory.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorFactory.java @@ -1,12 +1,15 @@ package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp; import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config.ConfigManager; +import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor.TextProcessor; +import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor.TextProcessorLocal; +import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor.TextProcessorService; public class TextProcessorFactory { public TextProcessor createCoreNlpTextProcessor() { if (ConfigManager.getInstance().getProperty("nlpProviderSource").equals("microservice") - && TextProcessorService.isMicroserviceAvailable()) { + && MicroserviceChecker.isMicroserviceAvailable()) { // return a text processor that uses the CoreNLP microservice return new TextProcessorService(); } else { diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessor.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java similarity index 93% rename from stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessor.java rename to stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java index 8d0ae19da..190fc45ec 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessor.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java @@ -1,4 +1,4 @@ -package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp; +package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor; import edu.kit.kastel.mcse.ardoco.core.api.text.Text; diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorLocal.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorLocal.java similarity index 93% rename from stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorLocal.java rename to stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorLocal.java index 9160560ce..b9623d79f 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorLocal.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorLocal.java @@ -1,6 +1,7 @@ -package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp; +package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor; import edu.kit.kastel.mcse.ardoco.core.api.text.Text; +import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.TextImpl; import edu.stanford.nlp.pipeline.CoreDocument; import edu.stanford.nlp.pipeline.StanfordCoreNLP; diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java similarity index 72% rename from stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java rename to stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java index 5b4999417..92b2451dd 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java @@ -1,4 +1,4 @@ -package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp; +package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor; import edu.kit.kastel.mcse.ardoco.core.api.text.Text; @@ -32,28 +32,6 @@ public Text processText(String inputText) { return new DtoToObjectConverter().convertText(textDto); } - /** - * checks if the CoreNLP microservice is available and can provide its services. - * @return whether the microservice is available - */ - public static boolean isMicroserviceAvailable() { - String requestUrl = ConfigManager.getInstance().getProperty("microserviceUrl") + ConfigManager.getInstance().getProperty("healthService"); - try { - URL url = new URL(requestUrl); - HttpURLConnection con = (HttpURLConnection) url.openConnection(); - con.setRequestMethod("GET"); - con.setConnectTimeout(5000); // timeout after 5 sec - int statusCode = con.getResponseCode(); - con.disconnect(); - if (statusCode != HttpURLConnection.HTTP_OK) { - return false; - } - } catch (Exception e) { - return false; - } - return true; - } - private String sendCorenlpRequest(String inputText) throws IOException { inputText = URLEncoder.encode(inputText, StandardCharsets.UTF_8); String requestUrl = ConfigManager.getInstance().getProperty("microserviceUrl") From 06bc011c426603a32d6343fb084009f4d43d3804 Mon Sep 17 00:00:00 2001 From: laura Date: Sat, 24 Jun 2023 15:06:56 +0200 Subject: [PATCH 10/29] removed TextProcessorFactory.java and added logger --- .../informants/corenlp/CoreNLPProvider.java | 4 +- .../informants/corenlp/TextProcessor.java | 51 +++++++++++++++++++ .../corenlp/TextProcessorFactory.java | 20 -------- .../TextProcessorLocal.java | 12 +++-- .../TextProcessorService.java | 22 ++++---- .../corenlp/textprocessor/TextProcessor.java | 14 ----- 6 files changed, 71 insertions(+), 52 deletions(-) create mode 100644 stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessor.java delete mode 100644 stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorFactory.java rename stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/{textprocessor => }/TextProcessorLocal.java (87%) rename stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/{textprocessor => }/TextProcessorService.java (79%) delete mode 100644 stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/CoreNLPProvider.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/CoreNLPProvider.java index 842e0c49e..6d94c985a 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/CoreNLPProvider.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/CoreNLPProvider.java @@ -8,7 +8,6 @@ import edu.kit.kastel.mcse.ardoco.core.api.text.Text; import edu.kit.kastel.mcse.ardoco.core.common.util.DataRepositoryHelper; import edu.kit.kastel.mcse.ardoco.core.data.DataRepository; -import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor.TextProcessor; public class CoreNLPProvider extends NlpInformant { @@ -53,8 +52,7 @@ public synchronized Text getAnnotatedText() { } private Text processText(String inputText) { - TextProcessor textProcessor = new TextProcessorFactory().createCoreNlpTextProcessor(); - return textProcessor.processText(inputText); + return new TextProcessor().processText(inputText); } @Override diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessor.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessor.java new file mode 100644 index 000000000..9e2e609a1 --- /dev/null +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessor.java @@ -0,0 +1,51 @@ +package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp; + +import edu.kit.kastel.mcse.ardoco.core.api.text.Text; +import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config.ConfigManager; + +import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This text processor processes texts using CoreNLP. + */ +public class TextProcessor { + + private final int maxFailedServiceRequests = 2; + Logger logger = LoggerFactory.getLogger(TextProcessor.class); + + /** + * processes and annotates a given text + * @param inputText the input text + * @return the annotated text + */ + public Text processText(String inputText) { + if (ConfigManager.getInstance().getProperty("nlpProviderSource").equals("microservice") + && MicroserviceChecker.isMicroserviceAvailable()) { + // return a text processor that uses the CoreNLP microservice + int k = 0; + while (k < maxFailedServiceRequests) { + try { + Text processedText = processService(inputText); + logger.info("Processed text with CoreNLP microservice."); + return processedText; + } catch (IOException e) { + k++; + } + } + logger.warn("Could not process text with CoreNLP microservice. Processing locally instead."); + } + logger.info("Processed text locally."); + return processLocally(inputText); + } + + private Text processLocally(String inputText) { + return new TextProcessorLocal().processText(inputText); + } + + private Text processService(String inputText) throws IOException { + return new TextProcessorService().processText(inputText); + } + +} diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorFactory.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorFactory.java deleted file mode 100644 index ed7f2d12e..000000000 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorFactory.java +++ /dev/null @@ -1,20 +0,0 @@ -package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp; - -import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config.ConfigManager; -import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor.TextProcessor; -import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor.TextProcessorLocal; -import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor.TextProcessorService; - -public class TextProcessorFactory { - - public TextProcessor createCoreNlpTextProcessor() { - if (ConfigManager.getInstance().getProperty("nlpProviderSource").equals("microservice") - && MicroserviceChecker.isMicroserviceAvailable()) { - // return a text processor that uses the CoreNLP microservice - return new TextProcessorService(); - } else { - return new TextProcessorLocal(); - } - } - -} diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorLocal.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorLocal.java similarity index 87% rename from stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorLocal.java rename to stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorLocal.java index b9623d79f..11b45ca44 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorLocal.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorLocal.java @@ -1,7 +1,6 @@ -package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor; +package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp; import edu.kit.kastel.mcse.ardoco.core.api.text.Text; -import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.TextImpl; import edu.stanford.nlp.pipeline.CoreDocument; import edu.stanford.nlp.pipeline.StanfordCoreNLP; @@ -10,11 +9,16 @@ /** * This text processor processes texts locally using CoreNLP. */ -public class TextProcessorLocal implements TextProcessor { +public class TextProcessorLocal { private static final String ANNOTATORS = "tokenize,ssplit,pos,parse,depparse,lemma"; // further: ",ner,coref" private static final String DEPENDENCIES_ANNOTATION = "EnhancedPlusPlusDependenciesAnnotation"; - @Override + /** + * processes and annotates a given text locally using CoreNLP. + * + * @param inputText the input text + * @return the annotated text + */ public Text processText(String inputText) { Properties props = getStanfordProperties(new Properties()); StanfordCoreNLP pipeline = new StanfordCoreNLP(props); diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java similarity index 79% rename from stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java rename to stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java index 92b2451dd..19a12f1b7 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java @@ -1,4 +1,4 @@ -package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor; +package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp; import edu.kit.kastel.mcse.ardoco.core.api.text.Text; @@ -18,17 +18,17 @@ /** * This text processor processes texts by sending requests to a microservice, which provides text processing using CoreNLP. */ -public class TextProcessorService implements TextProcessor { +public class TextProcessorService { - @Override - public Text processText(String inputText) { + /** + * processes and annotates a given text by sending requests to a microservice + * @param inputText the input text + * @return the annotated text + */ + public Text processText(String inputText) throws IOException { TextDTO textDto; - try { - String jsonText = sendCorenlpRequest(inputText); - textDto = JsonConverter.fromJsonString(jsonText); - } catch (IOException e) { - return null; // todo error handling - } + String jsonText = sendCorenlpRequest(inputText); + textDto = JsonConverter.fromJsonString(jsonText); return new DtoToObjectConverter().convertText(textDto); } @@ -45,7 +45,7 @@ private String sendGetRequest(String requestUrl) throws IOException { HttpURLConnection con = (HttpURLConnection) url.openConnection(); con.setRequestMethod("GET"); if (con.getResponseCode() != HttpURLConnection.HTTP_OK) { - return null; // TODO error handling + throw new IOException("HTTP error code: " + con.getResponseCode()); } BufferedReader in = new BufferedReader( new InputStreamReader(con.getInputStream()) diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java deleted file mode 100644 index 190fc45ec..000000000 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java +++ /dev/null @@ -1,14 +0,0 @@ -package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor; - -import edu.kit.kastel.mcse.ardoco.core.api.text.Text; - -public interface TextProcessor { - - /** - * processes and annotates a given text - * @param inputText the input text - * @return the annotated text - */ - Text processText(String inputText); - -} From c6b82dd91750d618af119010ed1643cfeaa4b450 Mon Sep 17 00:00:00 2001 From: laura Date: Sat, 1 Jul 2023 20:30:54 +0200 Subject: [PATCH 11/29] add authenticated microservice requests --- .../informants/corenlp/TextProcessor.java | 2 +- .../corenlp/TextProcessorService.java | 30 ++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessor.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessor.java index 9e2e609a1..f7a7b7eae 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessor.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessor.java @@ -23,7 +23,6 @@ public class TextProcessor { public Text processText(String inputText) { if (ConfigManager.getInstance().getProperty("nlpProviderSource").equals("microservice") && MicroserviceChecker.isMicroserviceAvailable()) { - // return a text processor that uses the CoreNLP microservice int k = 0; while (k < maxFailedServiceRequests) { try { @@ -32,6 +31,7 @@ public Text processText(String inputText) { return processedText; } catch (IOException e) { k++; + logger.warn("Could not process text with CoreNLP microservice. Trying again. Error: " + e.getMessage()); } } logger.warn("Could not process text with CoreNLP microservice. Processing locally instead."); diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java index 19a12f1b7..82bd5fbfb 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java @@ -14,6 +14,7 @@ import java.net.URL; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; +import java.util.Base64; /** * This text processor processes texts by sending requests to a microservice, which provides text processing using CoreNLP. @@ -37,13 +38,40 @@ private String sendCorenlpRequest(String inputText) throws IOException { String requestUrl = ConfigManager.getInstance().getProperty("microserviceUrl") + ConfigManager.getInstance().getProperty("corenlpService") + inputText; - return sendGetRequest(requestUrl); + return sendAuthenticatedGetRequest(requestUrl); } private String sendGetRequest(String requestUrl) throws IOException { URL url = new URL(requestUrl); HttpURLConnection con = (HttpURLConnection) url.openConnection(); con.setRequestMethod("GET"); + String content = readGetResponse(con); + con.disconnect(); + return content; + } + + public String sendAuthenticatedGetRequest(String requestUrl) throws IOException { + String username = System.getenv("USERNAME"); + String password = System.getenv("PASSWORD"); + if (username == null || password == null) { + throw new IOException("Environment variables USERNAME and PASSWORD must be set."); + } + URL url = new URL(requestUrl); + HttpURLConnection con = (HttpURLConnection) url.openConnection(); + con.setRequestMethod("GET"); + + // Encode the username and password + String authString = username + ":" + password; + String encodedAuthString = Base64.getEncoder().encodeToString(authString.getBytes()); + String authHeaderValue = "Basic " + encodedAuthString; + con.setRequestProperty("Authorization", authHeaderValue); + + String content = readGetResponse(con); + con.disconnect(); + return content; + } + + private String readGetResponse(HttpURLConnection con) throws IOException { if (con.getResponseCode() != HttpURLConnection.HTTP_OK) { throw new IOException("HTTP error code: " + con.getResponseCode()); } From 563906f68431d932d693581fbf2cc4eeb97dff5d Mon Sep 17 00:00:00 2001 From: laura Date: Fri, 11 Aug 2023 16:27:15 +0200 Subject: [PATCH 12/29] update MicroserviceChecker --- stages/text-preprocessing/pom.xml | 2 +- .../informants/corenlp/CoreNLPProvider.java | 1 + .../corenlp/MicroserviceChecker.java | 38 ---------------- .../informants/corenlp/TextImpl.java | 2 +- .../textprocessor/MicroserviceChecker.java | 45 +++++++++++++++++++ .../{ => textprocessor}/TextProcessor.java | 18 +++++--- .../TextProcessorLocal.java | 3 +- .../TextProcessorService.java | 2 +- 8 files changed, 64 insertions(+), 47 deletions(-) delete mode 100644 stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/MicroserviceChecker.java create mode 100644 stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java rename stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/{ => textprocessor}/TextProcessor.java (75%) rename stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/{ => textprocessor}/TextProcessorLocal.java (93%) rename stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/{ => textprocessor}/TextProcessorService.java (99%) diff --git a/stages/text-preprocessing/pom.xml b/stages/text-preprocessing/pom.xml index aa7c7fc54..0c82304f5 100644 --- a/stages/text-preprocessing/pom.xml +++ b/stages/text-preprocessing/pom.xml @@ -60,7 +60,7 @@ io.github.ardoco text-provider-json - 0.1.0 + 0.4.0 diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/CoreNLPProvider.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/CoreNLPProvider.java index 6d94c985a..6ad009b99 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/CoreNLPProvider.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/CoreNLPProvider.java @@ -8,6 +8,7 @@ import edu.kit.kastel.mcse.ardoco.core.api.text.Text; import edu.kit.kastel.mcse.ardoco.core.common.util.DataRepositoryHelper; import edu.kit.kastel.mcse.ardoco.core.data.DataRepository; +import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor.TextProcessor; public class CoreNLPProvider extends NlpInformant { diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/MicroserviceChecker.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/MicroserviceChecker.java deleted file mode 100644 index 2c3c56fcf..000000000 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/MicroserviceChecker.java +++ /dev/null @@ -1,38 +0,0 @@ -package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp; - -import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config.ConfigManager; - -import java.net.HttpURLConnection; -import java.net.URL; - -/** - * This utility class provides methods to check whether the microservice is available. - */ -public final class MicroserviceChecker { - - private MicroserviceChecker() { - } - - /** - * checks if the CoreNLP microservice is available and can provide its services. - * @return whether the microservice is available - */ - public static boolean isMicroserviceAvailable() { - String requestUrl = ConfigManager.getInstance().getProperty("microserviceUrl") + ConfigManager.getInstance().getProperty("healthService"); - try { - URL url = new URL(requestUrl); - HttpURLConnection con = (HttpURLConnection) url.openConnection(); - con.setRequestMethod("GET"); - con.setConnectTimeout(5000); // timeout after 5 sec - int statusCode = con.getResponseCode(); - con.disconnect(); - if (statusCode != HttpURLConnection.HTTP_OK) { - return false; - } - } catch (Exception e) { - return false; - } - return true; - } - -} diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextImpl.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextImpl.java index 22b7d989a..8f06559aa 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextImpl.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextImpl.java @@ -16,7 +16,7 @@ public class TextImpl implements Text { private ImmutableList sentences = Lists.immutable.empty(); private ImmutableList words = Lists.immutable.empty(); - TextImpl(CoreDocument coreDocument) { + public TextImpl(CoreDocument coreDocument) { this.coreDocument = coreDocument; } diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java new file mode 100644 index 000000000..5ddbf9b8a --- /dev/null +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java @@ -0,0 +1,45 @@ +package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor; + +import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config.ConfigManager; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.Base64; + +/** + * This utility class provides methods to check whether the microservice is available. + */ +public final class MicroserviceChecker { + + private MicroserviceChecker() { + } + + /** + * checks if the CoreNLP microservice is available and can provide its services. + * @return whether the microservice is available + */ + public static boolean isMicroserviceAvailable() throws IOException { + String requestUrl = ConfigManager.getInstance().getProperty("microserviceUrl") + ConfigManager.getInstance().getProperty("healthService"); + + String username = System.getenv("USERNAME"); + String password = System.getenv("PASSWORD"); + if (username == null || password == null) { + throw new IOException("Environment variables USERNAME and PASSWORD must be set."); + } + URL url = new URL(requestUrl); + HttpURLConnection con = (HttpURLConnection) url.openConnection(); + con.setRequestMethod("GET"); + con.setConnectTimeout(5000); // timeout after 5 sec + + // Encode the username and password + String authString = username + ":" + password; + String encodedAuthString = Base64.getEncoder().encodeToString(authString.getBytes()); + String authHeaderValue = "Basic " + encodedAuthString; + con.setRequestProperty("Authorization", authHeaderValue); + int statusCode = con.getResponseCode(); + con.disconnect(); + return statusCode == HttpURLConnection.HTTP_OK; + } + +} diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessor.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java similarity index 75% rename from stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessor.java rename to stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java index f7a7b7eae..688e2f194 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessor.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java @@ -1,9 +1,10 @@ -package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp; +package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor; import edu.kit.kastel.mcse.ardoco.core.api.text.Text; import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config.ConfigManager; import java.io.IOException; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -12,7 +13,7 @@ */ public class TextProcessor { - private final int maxFailedServiceRequests = 2; + private static final int MAX_FAILED_SERVICE_REQUESTS = 2; Logger logger = LoggerFactory.getLogger(TextProcessor.class); /** @@ -21,17 +22,24 @@ public class TextProcessor { * @return the annotated text */ public Text processText(String inputText) { + boolean microserviceAvailable; + try { + microserviceAvailable = MicroserviceChecker.isMicroserviceAvailable(); + } catch (IOException e) { + microserviceAvailable = false; + logger.warn("Could not check if CoreNLP microservice is available. ", e); + } if (ConfigManager.getInstance().getProperty("nlpProviderSource").equals("microservice") - && MicroserviceChecker.isMicroserviceAvailable()) { + && microserviceAvailable) { int k = 0; - while (k < maxFailedServiceRequests) { + while (k < MAX_FAILED_SERVICE_REQUESTS) { try { Text processedText = processService(inputText); logger.info("Processed text with CoreNLP microservice."); return processedText; } catch (IOException e) { k++; - logger.warn("Could not process text with CoreNLP microservice. Trying again. Error: " + e.getMessage()); + logger.warn("Could not process text with CoreNLP microservice. Trying again. ", e); } } logger.warn("Could not process text with CoreNLP microservice. Processing locally instead."); diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorLocal.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorLocal.java similarity index 93% rename from stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorLocal.java rename to stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorLocal.java index 11b45ca44..756210d19 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorLocal.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorLocal.java @@ -1,6 +1,7 @@ -package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp; +package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor; import edu.kit.kastel.mcse.ardoco.core.api.text.Text; +import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.TextImpl; import edu.stanford.nlp.pipeline.CoreDocument; import edu.stanford.nlp.pipeline.StanfordCoreNLP; diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java similarity index 99% rename from stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java rename to stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java index 82bd5fbfb..7bcbf3f42 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextProcessorService.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java @@ -1,4 +1,4 @@ -package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp; +package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor; import edu.kit.kastel.mcse.ardoco.core.api.text.Text; From c7c65bc0046efe6846b98ea08ef1cfe4186e66e8 Mon Sep 17 00:00:00 2001 From: laura Date: Fri, 11 Aug 2023 16:38:27 +0200 Subject: [PATCH 13/29] update version of textprovider-json --- stages/text-preprocessing/pom.xml | 2 +- .../informants/corenlp/textprocessor/TextProcessor.java | 7 ++++++- .../corenlp/textprocessor/TextProcessorService.java | 8 +++++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/stages/text-preprocessing/pom.xml b/stages/text-preprocessing/pom.xml index 0c82304f5..7d91599ea 100644 --- a/stages/text-preprocessing/pom.xml +++ b/stages/text-preprocessing/pom.xml @@ -60,7 +60,7 @@ io.github.ardoco text-provider-json - 0.4.0 + 0.8.0 diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java index 688e2f194..cb1c8826f 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java @@ -5,6 +5,8 @@ import java.io.IOException; +import io.github.ardoco.textproviderjson.error.InvalidJsonException; +import io.github.ardoco.textproviderjson.error.NotConvertableException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,6 +42,9 @@ public Text processText(String inputText) { } catch (IOException e) { k++; logger.warn("Could not process text with CoreNLP microservice. Trying again. ", e); + } catch (NotConvertableException | InvalidJsonException e) { + logger.warn("Could not process text with CoreNLP microservice. Text not convertable. ", e); + return processLocally(inputText); } } logger.warn("Could not process text with CoreNLP microservice. Processing locally instead."); @@ -52,7 +57,7 @@ private Text processLocally(String inputText) { return new TextProcessorLocal().processText(inputText); } - private Text processService(String inputText) throws IOException { + private Text processService(String inputText) throws IOException, NotConvertableException, InvalidJsonException { return new TextProcessorService().processText(inputText); } diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java index 7bcbf3f42..273699390 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java @@ -5,7 +5,9 @@ import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config.ConfigManager; import io.github.ardoco.textproviderjson.converter.JsonConverter; import io.github.ardoco.textproviderjson.converter.DtoToObjectConverter; -import io.github.ardoco.textproviderjson.dto.TextDTO; +import io.github.ardoco.textproviderjson.dto.TextDto; +import io.github.ardoco.textproviderjson.error.InvalidJsonException; +import io.github.ardoco.textproviderjson.error.NotConvertableException; import java.io.BufferedReader; import java.io.IOException; @@ -26,8 +28,8 @@ public class TextProcessorService { * @param inputText the input text * @return the annotated text */ - public Text processText(String inputText) throws IOException { - TextDTO textDto; + public Text processText(String inputText) throws IOException, InvalidJsonException, NotConvertableException { + TextDto textDto; String jsonText = sendCorenlpRequest(inputText); textDto = JsonConverter.fromJsonString(jsonText); return new DtoToObjectConverter().convertText(textDto); From 0671bc78d093c829e7eeb75a451f4defc8a813ab Mon Sep 17 00:00:00 2001 From: laura Date: Sun, 13 Aug 2023 18:18:25 +0200 Subject: [PATCH 14/29] update path of config.properties --- .../providers/informants/corenlp/config/ConfigManager.java | 6 +++--- .../corenlp/config => resources}/config.properties | 0 2 files changed, 3 insertions(+), 3 deletions(-) rename stages/text-preprocessing/src/main/{java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config => resources}/config.properties (100%) diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java index ce44ca164..4042652eb 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java @@ -1,7 +1,7 @@ package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config; -import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.util.Properties; /** @@ -10,11 +10,11 @@ public class ConfigManager { private static ConfigManager instance; private final Properties properties; - private final String filePath = "src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/config.properties"; + private static final String filePath = "config.properties"; private ConfigManager() { properties = new Properties(); - try (FileInputStream fileInputStream = new FileInputStream(filePath);) { + try (InputStream fileInputStream = ConfigManager.class.getClassLoader().getResourceAsStream(filePath);) { properties.load(fileInputStream); } catch (IOException e) { e.printStackTrace(); diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/config.properties b/stages/text-preprocessing/src/main/resources/config.properties similarity index 100% rename from stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/config.properties rename to stages/text-preprocessing/src/main/resources/config.properties From e5c0f2de9bc1cfa0d33d8b3919e658302bb70c73 Mon Sep 17 00:00:00 2001 From: laura Date: Mon, 14 Aug 2023 08:44:09 +0200 Subject: [PATCH 15/29] add usage of environment variables in ConfigManager --- .../providers/informants/corenlp/config/ConfigManager.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java index 4042652eb..2263609d0 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java @@ -19,6 +19,12 @@ private ConfigManager() { } catch (IOException e) { e.printStackTrace(); } + if (System.getenv("MICROSERVICE_URL") != null) { + properties.setProperty("microserviceUrl", System.getenv("MICROSERVICE_URL")); + } + if (System.getenv("NLP_PROVIDER_SOURCE") != null) { + properties.setProperty("nlpProviderSource", System.getenv("NLP_PROVIDER_SOURCE")); + } } public static ConfigManager getInstance() { From a3f7411a3533bf158cb559ef606ef77ef4d25d2a Mon Sep 17 00:00:00 2001 From: Laxraa Date: Mon, 14 Aug 2023 07:35:57 +0000 Subject: [PATCH 16/29] Apply formatting changes --- stages/text-preprocessing/pom.xml | 10 +++---- .../corenlp/config/ConfigManager.java | 13 +++++---- .../textprocessor/MicroserviceChecker.java | 8 +++-- .../corenlp/textprocessor/TextProcessor.java | 17 ++++++----- .../textprocessor/TextProcessorLocal.java | 5 ++-- .../textprocessor/TextProcessorService.java | 29 +++++++++---------- 6 files changed, 43 insertions(+), 39 deletions(-) diff --git a/stages/text-preprocessing/pom.xml b/stages/text-preprocessing/pom.xml index 7d91599ea..b6995b58d 100644 --- a/stages/text-preprocessing/pom.xml +++ b/stages/text-preprocessing/pom.xml @@ -37,6 +37,11 @@ ${stanford.corenlp.version} models + + io.github.ardoco + text-provider-json + 0.8.0 + io.github.ardoco.core common @@ -57,10 +62,5 @@ slf4j-simple test - - io.github.ardoco - text-provider-json - 0.8.0 - diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java index 2263609d0..0ca8246c0 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java @@ -1,3 +1,4 @@ +/* Licensed under MIT 2023. */ package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config; import java.io.IOException; @@ -36,8 +37,9 @@ public static ConfigManager getInstance() { /** * gets the value of the given key in the config file - * @param key the key - * @return the value + * + * @param key the key + * @return the value */ public String getProperty(String key) { return properties.getProperty(key); @@ -45,10 +47,11 @@ public String getProperty(String key) { /** * sets the value of the given key in the config file - * @param key the key - * @param value the new value + * + * @param key the key + * @param value the new value */ public void setProperty(String key, String value) { properties.setProperty(key, value); } -} \ No newline at end of file +} diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java index 5ddbf9b8a..8478678f1 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java @@ -1,12 +1,13 @@ +/* Licensed under MIT 2023. */ package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor; -import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config.ConfigManager; - import java.io.IOException; import java.net.HttpURLConnection; import java.net.URL; import java.util.Base64; +import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config.ConfigManager; + /** * This utility class provides methods to check whether the microservice is available. */ @@ -17,7 +18,8 @@ private MicroserviceChecker() { /** * checks if the CoreNLP microservice is available and can provide its services. - * @return whether the microservice is available + * + * @return whether the microservice is available */ public static boolean isMicroserviceAvailable() throws IOException { String requestUrl = ConfigManager.getInstance().getProperty("microserviceUrl") + ConfigManager.getInstance().getProperty("healthService"); diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java index cb1c8826f..733ef3868 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java @@ -1,15 +1,16 @@ +/* Licensed under MIT 2023. */ package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor; -import edu.kit.kastel.mcse.ardoco.core.api.text.Text; -import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config.ConfigManager; - import java.io.IOException; -import io.github.ardoco.textproviderjson.error.InvalidJsonException; -import io.github.ardoco.textproviderjson.error.NotConvertableException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import edu.kit.kastel.mcse.ardoco.core.api.text.Text; +import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config.ConfigManager; +import io.github.ardoco.textproviderjson.error.InvalidJsonException; +import io.github.ardoco.textproviderjson.error.NotConvertableException; + /** * This text processor processes texts using CoreNLP. */ @@ -20,8 +21,9 @@ public class TextProcessor { /** * processes and annotates a given text + * * @param inputText the input text - * @return the annotated text + * @return the annotated text */ public Text processText(String inputText) { boolean microserviceAvailable; @@ -31,8 +33,7 @@ public Text processText(String inputText) { microserviceAvailable = false; logger.warn("Could not check if CoreNLP microservice is available. ", e); } - if (ConfigManager.getInstance().getProperty("nlpProviderSource").equals("microservice") - && microserviceAvailable) { + if (ConfigManager.getInstance().getProperty("nlpProviderSource").equals("microservice") && microserviceAvailable) { int k = 0; while (k < MAX_FAILED_SERVICE_REQUESTS) { try { diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorLocal.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorLocal.java index 756210d19..f9a04dff8 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorLocal.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorLocal.java @@ -1,12 +1,13 @@ +/* Licensed under MIT 2023. */ package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor; +import java.util.Properties; + import edu.kit.kastel.mcse.ardoco.core.api.text.Text; import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.TextImpl; import edu.stanford.nlp.pipeline.CoreDocument; import edu.stanford.nlp.pipeline.StanfordCoreNLP; -import java.util.Properties; - /** * This text processor processes texts locally using CoreNLP. */ diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java index 273699390..a6ed61704 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java @@ -1,14 +1,6 @@ +/* Licensed under MIT 2023. */ package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor; -import edu.kit.kastel.mcse.ardoco.core.api.text.Text; - -import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config.ConfigManager; -import io.github.ardoco.textproviderjson.converter.JsonConverter; -import io.github.ardoco.textproviderjson.converter.DtoToObjectConverter; -import io.github.ardoco.textproviderjson.dto.TextDto; -import io.github.ardoco.textproviderjson.error.InvalidJsonException; -import io.github.ardoco.textproviderjson.error.NotConvertableException; - import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; @@ -18,6 +10,14 @@ import java.nio.charset.StandardCharsets; import java.util.Base64; +import edu.kit.kastel.mcse.ardoco.core.api.text.Text; +import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config.ConfigManager; +import io.github.ardoco.textproviderjson.converter.DtoToObjectConverter; +import io.github.ardoco.textproviderjson.converter.JsonConverter; +import io.github.ardoco.textproviderjson.dto.TextDto; +import io.github.ardoco.textproviderjson.error.InvalidJsonException; +import io.github.ardoco.textproviderjson.error.NotConvertableException; + /** * This text processor processes texts by sending requests to a microservice, which provides text processing using CoreNLP. */ @@ -25,8 +25,9 @@ public class TextProcessorService { /** * processes and annotates a given text by sending requests to a microservice + * * @param inputText the input text - * @return the annotated text + * @return the annotated text */ public Text processText(String inputText) throws IOException, InvalidJsonException, NotConvertableException { TextDto textDto; @@ -37,9 +38,7 @@ public Text processText(String inputText) throws IOException, InvalidJsonExcepti private String sendCorenlpRequest(String inputText) throws IOException { inputText = URLEncoder.encode(inputText, StandardCharsets.UTF_8); - String requestUrl = ConfigManager.getInstance().getProperty("microserviceUrl") - + ConfigManager.getInstance().getProperty("corenlpService") - + inputText; + String requestUrl = ConfigManager.getInstance().getProperty("microserviceUrl") + ConfigManager.getInstance().getProperty("corenlpService") + inputText; return sendAuthenticatedGetRequest(requestUrl); } @@ -77,9 +76,7 @@ private String readGetResponse(HttpURLConnection con) throws IOException { if (con.getResponseCode() != HttpURLConnection.HTTP_OK) { throw new IOException("HTTP error code: " + con.getResponseCode()); } - BufferedReader in = new BufferedReader( - new InputStreamReader(con.getInputStream()) - ); + BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream())); String inputLine; StringBuilder content = new StringBuilder(); while ((inputLine = in.readLine()) != null) { From 210ae2d1d28b5ebb1a89c9a2f07e3971535f45e2 Mon Sep 17 00:00:00 2001 From: laura Date: Mon, 14 Aug 2023 14:06:05 +0200 Subject: [PATCH 17/29] remove code smells --- .../corenlp/config/ConfigManager.java | 17 ++++++++++++++--- .../textprocessor/TextProcessorService.java | 9 --------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java index 2263609d0..9e3c885db 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java @@ -1,5 +1,9 @@ package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config; +import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor.TextProcessor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.IOException; import java.io.InputStream; import java.util.Properties; @@ -8,16 +12,23 @@ * This Singleton manages access to the config file. */ public class ConfigManager { + + Logger logger = LoggerFactory.getLogger(ConfigManager.class); + private static ConfigManager instance; private final Properties properties; - private static final String filePath = "config.properties"; + private static final String FILE_PATH = "config.properties"; private ConfigManager() { properties = new Properties(); - try (InputStream fileInputStream = ConfigManager.class.getClassLoader().getResourceAsStream(filePath);) { + try (InputStream fileInputStream = ConfigManager.class.getClassLoader().getResourceAsStream(FILE_PATH);) { properties.load(fileInputStream); } catch (IOException e) { - e.printStackTrace(); + logger.warn("Could not load config file. ", e); + properties.setProperty("microserviceUrl", "http://localhost:8080"); + properties.setProperty("nlpProviderSource", "local"); + properties.setProperty("corenlpService", "/stanfordnlp?text="); + properties.setProperty("healthService", "/health"); } if (System.getenv("MICROSERVICE_URL") != null) { properties.setProperty("microserviceUrl", System.getenv("MICROSERVICE_URL")); diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java index 273699390..5d0f6f190 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java @@ -43,15 +43,6 @@ private String sendCorenlpRequest(String inputText) throws IOException { return sendAuthenticatedGetRequest(requestUrl); } - private String sendGetRequest(String requestUrl) throws IOException { - URL url = new URL(requestUrl); - HttpURLConnection con = (HttpURLConnection) url.openConnection(); - con.setRequestMethod("GET"); - String content = readGetResponse(con); - con.disconnect(); - return content; - } - public String sendAuthenticatedGetRequest(String requestUrl) throws IOException { String username = System.getenv("USERNAME"); String password = System.getenv("PASSWORD"); From ceb2ba29801ba5197871d44eadf5b57913c08947 Mon Sep 17 00:00:00 2001 From: Laxraa Date: Mon, 14 Aug 2023 12:08:07 +0000 Subject: [PATCH 18/29] Apply formatting changes --- .../providers/informants/corenlp/config/ConfigManager.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java index 6e162ff30..290bbe38e 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java @@ -1,14 +1,13 @@ /* Licensed under MIT 2023. */ package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config; -import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor.TextProcessor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.io.InputStream; import java.util.Properties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * This Singleton manages access to the config file. */ From 46804a585d509eabf9338021c58232959cafb477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Fuch=C3=9F?= Date: Mon, 14 Aug 2023 16:40:33 +0200 Subject: [PATCH 19/29] Fixed paths to health endpoint --- .../text/providers/informants/corenlp/config/ConfigManager.java | 2 +- stages/text-preprocessing/src/main/resources/config.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java index 290bbe38e..8d1c8bc31 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java @@ -28,7 +28,7 @@ private ConfigManager() { properties.setProperty("microserviceUrl", "http://localhost:8080"); properties.setProperty("nlpProviderSource", "local"); properties.setProperty("corenlpService", "/stanfordnlp?text="); - properties.setProperty("healthService", "/health"); + properties.setProperty("healthService", "/stanfordnlp/health"); } if (System.getenv("MICROSERVICE_URL") != null) { properties.setProperty("microserviceUrl", System.getenv("MICROSERVICE_URL")); diff --git a/stages/text-preprocessing/src/main/resources/config.properties b/stages/text-preprocessing/src/main/resources/config.properties index bfb9ab9cb..e25bff7ea 100644 --- a/stages/text-preprocessing/src/main/resources/config.properties +++ b/stages/text-preprocessing/src/main/resources/config.properties @@ -1,5 +1,5 @@ nlpProviderSource=microservice microserviceUrl= http://localhost:8080 corenlpService=/stanfordnlp?text= -healthService=/health +healthService=/stanfordnlp/health From fe9e55f3939615be35f2ca3690adf61b0af07998 Mon Sep 17 00:00:00 2001 From: laura Date: Tue, 22 Aug 2023 21:25:46 +0200 Subject: [PATCH 20/29] replace the magic strings by proper getters in the ConfigManager --- .../corenlp/config/ConfigManager.java | 45 ++++++++++--------- .../textprocessor/MicroserviceChecker.java | 2 +- .../corenlp/textprocessor/TextProcessor.java | 2 +- .../textprocessor/TextProcessorService.java | 2 +- 4 files changed, 26 insertions(+), 25 deletions(-) diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java index 8d1c8bc31..343506c94 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java @@ -18,6 +18,10 @@ public class ConfigManager { private static ConfigManager instance; private final Properties properties; private static final String FILE_PATH = "config.properties"; + private static final String PROPERTY_MICROSERVICE_URL = "microserviceUrl"; + private static final String PROPERTY_NLP_PROVIDER_SOURCE = "nlpProviderSource"; + private static final String PROPERTY_CORENLP_SERVICE = "corenlpService"; + private static final String PROPERTY_HEALTH_SERVICE = "healthService"; private ConfigManager() { properties = new Properties(); @@ -25,16 +29,16 @@ private ConfigManager() { properties.load(fileInputStream); } catch (IOException e) { logger.warn("Could not load config file. ", e); - properties.setProperty("microserviceUrl", "http://localhost:8080"); - properties.setProperty("nlpProviderSource", "local"); - properties.setProperty("corenlpService", "/stanfordnlp?text="); - properties.setProperty("healthService", "/stanfordnlp/health"); + properties.setProperty(PROPERTY_MICROSERVICE_URL, "http://localhost:8080"); + properties.setProperty(PROPERTY_NLP_PROVIDER_SOURCE, "local"); + properties.setProperty(PROPERTY_CORENLP_SERVICE, "/stanfordnlp?text="); + properties.setProperty(PROPERTY_HEALTH_SERVICE, "/stanfordnlp/health"); } if (System.getenv("MICROSERVICE_URL") != null) { - properties.setProperty("microserviceUrl", System.getenv("MICROSERVICE_URL")); + properties.setProperty(PROPERTY_MICROSERVICE_URL, System.getenv("MICROSERVICE_URL")); } if (System.getenv("NLP_PROVIDER_SOURCE") != null) { - properties.setProperty("nlpProviderSource", System.getenv("NLP_PROVIDER_SOURCE")); + properties.setProperty(PROPERTY_NLP_PROVIDER_SOURCE, System.getenv("NLP_PROVIDER_SOURCE")); } } @@ -45,23 +49,20 @@ public static ConfigManager getInstance() { return instance; } - /** - * gets the value of the given key in the config file - * - * @param key the key - * @return the value - */ - public String getProperty(String key) { - return properties.getProperty(key); + public String getMicroserviceUrl() { + return properties.getProperty(PROPERTY_MICROSERVICE_URL); } - /** - * sets the value of the given key in the config file - * - * @param key the key - * @param value the new value - */ - public void setProperty(String key, String value) { - properties.setProperty(key, value); + public String getNlpProviderSource() { + return properties.getProperty(PROPERTY_NLP_PROVIDER_SOURCE); } + + public String getCorenlpService() { + return properties.getProperty(PROPERTY_CORENLP_SERVICE); + } + + public String getHealthService() { + return properties.getProperty(PROPERTY_HEALTH_SERVICE); + } + } diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java index 8478678f1..934b594d1 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java @@ -22,7 +22,7 @@ private MicroserviceChecker() { * @return whether the microservice is available */ public static boolean isMicroserviceAvailable() throws IOException { - String requestUrl = ConfigManager.getInstance().getProperty("microserviceUrl") + ConfigManager.getInstance().getProperty("healthService"); + String requestUrl = ConfigManager.getInstance().getMicroserviceUrl() + ConfigManager.getInstance().getHealthService(); String username = System.getenv("USERNAME"); String password = System.getenv("PASSWORD"); diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java index 733ef3868..ba9ef5905 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java @@ -33,7 +33,7 @@ public Text processText(String inputText) { microserviceAvailable = false; logger.warn("Could not check if CoreNLP microservice is available. ", e); } - if (ConfigManager.getInstance().getProperty("nlpProviderSource").equals("microservice") && microserviceAvailable) { + if (ConfigManager.getInstance().getNlpProviderSource().equals("microservice") && microserviceAvailable) { int k = 0; while (k < MAX_FAILED_SERVICE_REQUESTS) { try { diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java index ef9d2f745..11b219d5b 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java @@ -38,7 +38,7 @@ public Text processText(String inputText) throws IOException, InvalidJsonExcepti private String sendCorenlpRequest(String inputText) throws IOException { inputText = URLEncoder.encode(inputText, StandardCharsets.UTF_8); - String requestUrl = ConfigManager.getInstance().getProperty("microserviceUrl") + ConfigManager.getInstance().getProperty("corenlpService") + inputText; + String requestUrl = ConfigManager.getInstance().getMicroserviceUrl() + ConfigManager.getInstance().getCorenlpService() + inputText; return sendAuthenticatedGetRequest(requestUrl); } From d0c51f7b825ec7676dbab3d1645796432f2723bd Mon Sep 17 00:00:00 2001 From: laura Date: Tue, 22 Aug 2023 21:27:30 +0200 Subject: [PATCH 21/29] give variable a better name in TextProcessor --- .../informants/corenlp/textprocessor/TextProcessor.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java index ba9ef5905..e3b6a28d6 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java @@ -34,14 +34,14 @@ public Text processText(String inputText) { logger.warn("Could not check if CoreNLP microservice is available. ", e); } if (ConfigManager.getInstance().getNlpProviderSource().equals("microservice") && microserviceAvailable) { - int k = 0; - while (k < MAX_FAILED_SERVICE_REQUESTS) { + int numberOfTry = 0; + while (numberOfTry < MAX_FAILED_SERVICE_REQUESTS) { try { Text processedText = processService(inputText); logger.info("Processed text with CoreNLP microservice."); return processedText; } catch (IOException e) { - k++; + numberOfTry++; logger.warn("Could not process text with CoreNLP microservice. Trying again. ", e); } catch (NotConvertableException | InvalidJsonException e) { logger.warn("Could not process text with CoreNLP microservice. Text not convertable. ", e); From acce8945447eac523653380ab30e263cb149a746 Mon Sep 17 00:00:00 2001 From: laura Date: Wed, 23 Aug 2023 14:53:44 +0200 Subject: [PATCH 22/29] use Apache HttpClients and add HttpCommunicator --- stages/text-preprocessing/pom.xml | 5 ++ .../textprocessor/HttpCommunicator.java | 60 +++++++++++++++++++ .../textprocessor/MicroserviceChecker.java | 26 ++------ .../textprocessor/TextProcessorService.java | 46 +++----------- 4 files changed, 78 insertions(+), 59 deletions(-) create mode 100644 stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/HttpCommunicator.java diff --git a/stages/text-preprocessing/pom.xml b/stages/text-preprocessing/pom.xml index b6995b58d..3b0a2803c 100644 --- a/stages/text-preprocessing/pom.xml +++ b/stages/text-preprocessing/pom.xml @@ -62,5 +62,10 @@ slf4j-simple test + + org.apache.httpcomponents + httpclient + 4.5.13 + diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/HttpCommunicator.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/HttpCommunicator.java new file mode 100644 index 000000000..7b068e949 --- /dev/null +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/HttpCommunicator.java @@ -0,0 +1,60 @@ +package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor; + +import org.apache.http.HttpEntity; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; + +public class HttpCommunicator { + + /** + * The returned CloseableHttpResponse must be closed after usage. + * @param requestUrl The URL to send the GET request to. + * @return The response of the GET request. + * @throws IOException If the request fails. + */ + public CloseableHttpResponse sendAuthenticatedGetRequest(String requestUrl) throws IOException { + String username = System.getenv("USERNAME"); + String password = System.getenv("PASSWORD"); + if (username == null || password == null) { + throw new IOException("Environment variables USERNAME and PASSWORD must be set."); + } + + HttpGet request = new HttpGet(requestUrl); + CredentialsProvider provider = new BasicCredentialsProvider(); + provider.setCredentials( + AuthScope.ANY, + new UsernamePasswordCredentials(username, password) + ); + + CloseableHttpClient httpClient = HttpClientBuilder.create() + .setDefaultCredentialsProvider(provider) + .build(); + return httpClient.execute(request); + } + + public String readGetResponse(CloseableHttpResponse response) throws IOException { + if (response.getStatusLine().getStatusCode() != HttpURLConnection.HTTP_OK) { + throw new IOException("HTTP error code: " + response.getStatusLine().getStatusCode()); + } + HttpEntity entity = response.getEntity(); + BufferedReader in = new BufferedReader(new InputStreamReader(entity.getContent())); + String inputLine; + StringBuilder content = new StringBuilder(); + while ((inputLine = in.readLine()) != null) { + content.append(inputLine); + } + in.close(); + return content.toString(); + } +} diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java index 934b594d1..e917e53dc 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java @@ -3,11 +3,11 @@ import java.io.IOException; import java.net.HttpURLConnection; -import java.net.URL; -import java.util.Base64; import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config.ConfigManager; +import org.apache.http.client.methods.CloseableHttpResponse; + /** * This utility class provides methods to check whether the microservice is available. */ @@ -23,25 +23,9 @@ private MicroserviceChecker() { */ public static boolean isMicroserviceAvailable() throws IOException { String requestUrl = ConfigManager.getInstance().getMicroserviceUrl() + ConfigManager.getInstance().getHealthService(); - - String username = System.getenv("USERNAME"); - String password = System.getenv("PASSWORD"); - if (username == null || password == null) { - throw new IOException("Environment variables USERNAME and PASSWORD must be set."); - } - URL url = new URL(requestUrl); - HttpURLConnection con = (HttpURLConnection) url.openConnection(); - con.setRequestMethod("GET"); - con.setConnectTimeout(5000); // timeout after 5 sec - - // Encode the username and password - String authString = username + ":" + password; - String encodedAuthString = Base64.getEncoder().encodeToString(authString.getBytes()); - String authHeaderValue = "Basic " + encodedAuthString; - con.setRequestProperty("Authorization", authHeaderValue); - int statusCode = con.getResponseCode(); - con.disconnect(); - return statusCode == HttpURLConnection.HTTP_OK; + CloseableHttpResponse response = new HttpCommunicator().sendAuthenticatedGetRequest(requestUrl); + response.close(); + return response.getStatusLine().getStatusCode() == HttpURLConnection.HTTP_OK; } } diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java index 11b219d5b..8a3085d8b 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java @@ -1,14 +1,11 @@ /* Licensed under MIT 2023. */ package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor; -import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStreamReader; -import java.net.HttpURLConnection; -import java.net.URL; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; -import java.util.Base64; + +import org.apache.http.client.methods.CloseableHttpResponse; import edu.kit.kastel.mcse.ardoco.core.api.text.Text; import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config.ConfigManager; @@ -42,38 +39,11 @@ private String sendCorenlpRequest(String inputText) throws IOException { return sendAuthenticatedGetRequest(requestUrl); } - public String sendAuthenticatedGetRequest(String requestUrl) throws IOException { - String username = System.getenv("USERNAME"); - String password = System.getenv("PASSWORD"); - if (username == null || password == null) { - throw new IOException("Environment variables USERNAME and PASSWORD must be set."); - } - URL url = new URL(requestUrl); - HttpURLConnection con = (HttpURLConnection) url.openConnection(); - con.setRequestMethod("GET"); - - // Encode the username and password - String authString = username + ":" + password; - String encodedAuthString = Base64.getEncoder().encodeToString(authString.getBytes()); - String authHeaderValue = "Basic " + encodedAuthString; - con.setRequestProperty("Authorization", authHeaderValue); - - String content = readGetResponse(con); - con.disconnect(); - return content; - } - - private String readGetResponse(HttpURLConnection con) throws IOException { - if (con.getResponseCode() != HttpURLConnection.HTTP_OK) { - throw new IOException("HTTP error code: " + con.getResponseCode()); - } - BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream())); - String inputLine; - StringBuilder content = new StringBuilder(); - while ((inputLine = in.readLine()) != null) { - content.append(inputLine); - } - in.close(); - return content.toString(); + private String sendAuthenticatedGetRequest(String requestUrl) throws IOException { + HttpCommunicator httpCommunicator = new HttpCommunicator(); + CloseableHttpResponse response = httpCommunicator.sendAuthenticatedGetRequest(requestUrl); + String responseString = httpCommunicator.readGetResponse(response); + response.close(); + return responseString; } } From c3f0f6198330ceaa0518ae33d4ca24defdde07b3 Mon Sep 17 00:00:00 2001 From: laura Date: Wed, 23 Aug 2023 17:27:55 +0200 Subject: [PATCH 23/29] refactor HttpCommunicator --- stages/text-preprocessing/pom.xml | 6 +- .../textprocessor/CustomHttpResponse.java | 4 ++ .../textprocessor/CustomResponseHandler.java | 33 +++++++++++ .../textprocessor/HttpCommunicator.java | 59 +++++-------------- .../textprocessor/MicroserviceChecker.java | 7 +-- .../textprocessor/TextProcessorService.java | 8 +-- 6 files changed, 59 insertions(+), 58 deletions(-) create mode 100644 stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/CustomHttpResponse.java create mode 100644 stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/CustomResponseHandler.java diff --git a/stages/text-preprocessing/pom.xml b/stages/text-preprocessing/pom.xml index 3b0a2803c..2fd473007 100644 --- a/stages/text-preprocessing/pom.xml +++ b/stages/text-preprocessing/pom.xml @@ -55,7 +55,6 @@ org.slf4j log4j-over-slf4j - ${slf4j.version} org.slf4j @@ -63,9 +62,8 @@ test - org.apache.httpcomponents - httpclient - 4.5.13 + org.apache.httpcomponents.client5 + httpclient5 diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/CustomHttpResponse.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/CustomHttpResponse.java new file mode 100644 index 000000000..b085142b1 --- /dev/null +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/CustomHttpResponse.java @@ -0,0 +1,4 @@ +package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor; + +public record CustomHttpResponse (String responseBody, int statusCode){ +} diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/CustomResponseHandler.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/CustomResponseHandler.java new file mode 100644 index 000000000..40b5e76c2 --- /dev/null +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/CustomResponseHandler.java @@ -0,0 +1,33 @@ +package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor; + +import org.apache.hc.core5.http.ClassicHttpResponse; +import org.apache.hc.core5.http.HttpEntity; +import org.apache.hc.core5.http.io.HttpClientResponseHandler; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; + +public class CustomResponseHandler implements HttpClientResponseHandler { + @Override + public CustomHttpResponse handleResponse(ClassicHttpResponse response) throws IOException { + return new CustomHttpResponse(readGetResponse(response), response.getCode()); + } + + + private String readGetResponse(ClassicHttpResponse response) throws IOException { + if (response.getCode() != HttpURLConnection.HTTP_OK) { + throw new IOException("HTTP error code: " + response.getCode()); + } + HttpEntity entity = response.getEntity(); + BufferedReader in = new BufferedReader(new InputStreamReader(entity.getContent())); + String inputLine; + StringBuilder content = new StringBuilder(); + while ((inputLine = in.readLine()) != null) { + content.append(inputLine); + } + in.close(); + return content.toString(); + } +} diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/HttpCommunicator.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/HttpCommunicator.java index 7b068e949..5017364dc 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/HttpCommunicator.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/HttpCommunicator.java @@ -1,29 +1,19 @@ package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor; -import org.apache.http.HttpEntity; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; - -import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStreamReader; -import java.net.HttpURLConnection; + +import org.apache.hc.client5.http.auth.AuthScope; +import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.classic.methods.HttpGet; +import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClients; +import org.apache.hc.core5.http.io.HttpClientResponseHandler; public class HttpCommunicator { - /** - * The returned CloseableHttpResponse must be closed after usage. - * @param requestUrl The URL to send the GET request to. - * @return The response of the GET request. - * @throws IOException If the request fails. - */ - public CloseableHttpResponse sendAuthenticatedGetRequest(String requestUrl) throws IOException { + + public CustomHttpResponse sendAuthenticatedGetRequest(String requestUrl) throws IOException { String username = System.getenv("USERNAME"); String password = System.getenv("PASSWORD"); if (username == null || password == null) { @@ -31,30 +21,13 @@ public CloseableHttpResponse sendAuthenticatedGetRequest(String requestUrl) thro } HttpGet request = new HttpGet(requestUrl); - CredentialsProvider provider = new BasicCredentialsProvider(); + BasicCredentialsProvider provider = new BasicCredentialsProvider(); provider.setCredentials( - AuthScope.ANY, - new UsernamePasswordCredentials(username, password) - ); - - CloseableHttpClient httpClient = HttpClientBuilder.create() - .setDefaultCredentialsProvider(provider) - .build(); - return httpClient.execute(request); - } - - public String readGetResponse(CloseableHttpResponse response) throws IOException { - if (response.getStatusLine().getStatusCode() != HttpURLConnection.HTTP_OK) { - throw new IOException("HTTP error code: " + response.getStatusLine().getStatusCode()); - } - HttpEntity entity = response.getEntity(); - BufferedReader in = new BufferedReader(new InputStreamReader(entity.getContent())); - String inputLine; - StringBuilder content = new StringBuilder(); - while ((inputLine = in.readLine()) != null) { - content.append(inputLine); + new AuthScope(null, -1), + new UsernamePasswordCredentials(username, password.toCharArray())); + HttpClientResponseHandler responseHandler = new CustomResponseHandler(); + try (CloseableHttpClient httpClient = HttpClients.custom().setDefaultCredentialsProvider(provider).build()) { + return httpClient.execute(request, responseHandler); } - in.close(); - return content.toString(); } } diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java index e917e53dc..bd47b218c 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java @@ -6,8 +6,6 @@ import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config.ConfigManager; -import org.apache.http.client.methods.CloseableHttpResponse; - /** * This utility class provides methods to check whether the microservice is available. */ @@ -23,9 +21,8 @@ private MicroserviceChecker() { */ public static boolean isMicroserviceAvailable() throws IOException { String requestUrl = ConfigManager.getInstance().getMicroserviceUrl() + ConfigManager.getInstance().getHealthService(); - CloseableHttpResponse response = new HttpCommunicator().sendAuthenticatedGetRequest(requestUrl); - response.close(); - return response.getStatusLine().getStatusCode() == HttpURLConnection.HTTP_OK; + CustomHttpResponse response = new HttpCommunicator().sendAuthenticatedGetRequest(requestUrl); + return response.statusCode() == HttpURLConnection.HTTP_OK; } } diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java index 8a3085d8b..c086acf61 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java @@ -5,8 +5,6 @@ import java.net.URLEncoder; import java.nio.charset.StandardCharsets; -import org.apache.http.client.methods.CloseableHttpResponse; - import edu.kit.kastel.mcse.ardoco.core.api.text.Text; import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config.ConfigManager; import io.github.ardoco.textproviderjson.converter.DtoToObjectConverter; @@ -41,9 +39,7 @@ private String sendCorenlpRequest(String inputText) throws IOException { private String sendAuthenticatedGetRequest(String requestUrl) throws IOException { HttpCommunicator httpCommunicator = new HttpCommunicator(); - CloseableHttpResponse response = httpCommunicator.sendAuthenticatedGetRequest(requestUrl); - String responseString = httpCommunicator.readGetResponse(response); - response.close(); - return responseString; + CustomHttpResponse response = httpCommunicator.sendAuthenticatedGetRequest(requestUrl); + return response.responseBody(); } } From 719b5dddf7db95f927cfe278736ce3138fc79a3d Mon Sep 17 00:00:00 2001 From: laura Date: Wed, 23 Aug 2023 17:31:59 +0200 Subject: [PATCH 24/29] use dependency management --- pom.xml | 5 +++++ stages/text-preprocessing/pom.xml | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7ce4c6579..355a17c17 100644 --- a/pom.xml +++ b/pom.xml @@ -170,6 +170,11 @@ eclipse-collections ${eclipse-collections.version} + + io.github.ardoco + text-provider-json + 0.8.0 + diff --git a/stages/text-preprocessing/pom.xml b/stages/text-preprocessing/pom.xml index 2fd473007..4d5bd50c0 100644 --- a/stages/text-preprocessing/pom.xml +++ b/stages/text-preprocessing/pom.xml @@ -40,7 +40,6 @@ io.github.ardoco text-provider-json - 0.8.0 io.github.ardoco.core From 01c399d230849fbb4018396f26d08c91cef908f9 Mon Sep 17 00:00:00 2001 From: laura Date: Sun, 27 Aug 2023 16:41:39 +0200 Subject: [PATCH 25/29] use BasicHttpClientResponseHandler --- .../textprocessor/CustomHttpResponse.java | 4 --- .../textprocessor/CustomResponseHandler.java | 33 ------------------- .../textprocessor/HttpCommunicator.java | 7 ++-- .../textprocessor/MicroserviceChecker.java | 5 ++- .../textprocessor/TextProcessorService.java | 3 +- 5 files changed, 6 insertions(+), 46 deletions(-) delete mode 100644 stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/CustomHttpResponse.java delete mode 100644 stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/CustomResponseHandler.java diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/CustomHttpResponse.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/CustomHttpResponse.java deleted file mode 100644 index b085142b1..000000000 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/CustomHttpResponse.java +++ /dev/null @@ -1,4 +0,0 @@ -package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor; - -public record CustomHttpResponse (String responseBody, int statusCode){ -} diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/CustomResponseHandler.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/CustomResponseHandler.java deleted file mode 100644 index 40b5e76c2..000000000 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/CustomResponseHandler.java +++ /dev/null @@ -1,33 +0,0 @@ -package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor; - -import org.apache.hc.core5.http.ClassicHttpResponse; -import org.apache.hc.core5.http.HttpEntity; -import org.apache.hc.core5.http.io.HttpClientResponseHandler; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.HttpURLConnection; - -public class CustomResponseHandler implements HttpClientResponseHandler { - @Override - public CustomHttpResponse handleResponse(ClassicHttpResponse response) throws IOException { - return new CustomHttpResponse(readGetResponse(response), response.getCode()); - } - - - private String readGetResponse(ClassicHttpResponse response) throws IOException { - if (response.getCode() != HttpURLConnection.HTTP_OK) { - throw new IOException("HTTP error code: " + response.getCode()); - } - HttpEntity entity = response.getEntity(); - BufferedReader in = new BufferedReader(new InputStreamReader(entity.getContent())); - String inputLine; - StringBuilder content = new StringBuilder(); - while ((inputLine = in.readLine()) != null) { - content.append(inputLine); - } - in.close(); - return content.toString(); - } -} diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/HttpCommunicator.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/HttpCommunicator.java index 5017364dc..d2b248a6e 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/HttpCommunicator.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/HttpCommunicator.java @@ -6,14 +6,14 @@ import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; +import org.apache.hc.client5.http.impl.classic.BasicHttpClientResponseHandler; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.HttpClients; -import org.apache.hc.core5.http.io.HttpClientResponseHandler; public class HttpCommunicator { - public CustomHttpResponse sendAuthenticatedGetRequest(String requestUrl) throws IOException { + public String sendAuthenticatedGetRequest(String requestUrl) throws IOException { String username = System.getenv("USERNAME"); String password = System.getenv("PASSWORD"); if (username == null || password == null) { @@ -25,9 +25,8 @@ public CustomHttpResponse sendAuthenticatedGetRequest(String requestUrl) throws provider.setCredentials( new AuthScope(null, -1), new UsernamePasswordCredentials(username, password.toCharArray())); - HttpClientResponseHandler responseHandler = new CustomResponseHandler(); try (CloseableHttpClient httpClient = HttpClients.custom().setDefaultCredentialsProvider(provider).build()) { - return httpClient.execute(request, responseHandler); + return httpClient.execute(request, new BasicHttpClientResponseHandler()); } } } diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java index bd47b218c..5dd0f219b 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java @@ -2,7 +2,6 @@ package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor; import java.io.IOException; -import java.net.HttpURLConnection; import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config.ConfigManager; @@ -21,8 +20,8 @@ private MicroserviceChecker() { */ public static boolean isMicroserviceAvailable() throws IOException { String requestUrl = ConfigManager.getInstance().getMicroserviceUrl() + ConfigManager.getInstance().getHealthService(); - CustomHttpResponse response = new HttpCommunicator().sendAuthenticatedGetRequest(requestUrl); - return response.statusCode() == HttpURLConnection.HTTP_OK; + String response = new HttpCommunicator().sendAuthenticatedGetRequest(requestUrl); + return response.equals("Microservice is healthy"); } } diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java index c086acf61..89e0d91a0 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java @@ -39,7 +39,6 @@ private String sendCorenlpRequest(String inputText) throws IOException { private String sendAuthenticatedGetRequest(String requestUrl) throws IOException { HttpCommunicator httpCommunicator = new HttpCommunicator(); - CustomHttpResponse response = httpCommunicator.sendAuthenticatedGetRequest(requestUrl); - return response.responseBody(); + return httpCommunicator.sendAuthenticatedGetRequest(requestUrl); } } From 9a9c7bf71960c62f1f4f5c99e51d2ef54cba145f Mon Sep 17 00:00:00 2001 From: Jan Keim Date: Mon, 28 Aug 2023 15:08:21 +0200 Subject: [PATCH 26/29] Format --- pom.xml | 10 +++++----- stages/text-preprocessing/pom.xml | 8 ++++---- .../corenlp/textprocessor/HttpCommunicator.java | 6 ++---- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/pom.xml b/pom.xml index 355a17c17..f6d7ec73c 100644 --- a/pom.xml +++ b/pom.xml @@ -141,6 +141,11 @@ commons-io 2.13.0 + + io.github.ardoco + text-provider-json + 0.8.0 + @@ -170,11 +175,6 @@ eclipse-collections ${eclipse-collections.version} - - io.github.ardoco - text-provider-json - 0.8.0 - diff --git a/stages/text-preprocessing/pom.xml b/stages/text-preprocessing/pom.xml index 4d5bd50c0..badcfd41e 100644 --- a/stages/text-preprocessing/pom.xml +++ b/stages/text-preprocessing/pom.xml @@ -46,6 +46,10 @@ common ${revision} + + org.apache.httpcomponents.client5 + httpclient5 + org.junit.jupiter junit-jupiter-params @@ -60,9 +64,5 @@ slf4j-simple test - - org.apache.httpcomponents.client5 - httpclient5 - diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/HttpCommunicator.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/HttpCommunicator.java index d2b248a6e..4aa468dac 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/HttpCommunicator.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/HttpCommunicator.java @@ -1,3 +1,4 @@ +/* Licensed under MIT 2023. */ package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor; import java.io.IOException; @@ -12,7 +13,6 @@ public class HttpCommunicator { - public String sendAuthenticatedGetRequest(String requestUrl) throws IOException { String username = System.getenv("USERNAME"); String password = System.getenv("PASSWORD"); @@ -22,9 +22,7 @@ public String sendAuthenticatedGetRequest(String requestUrl) throws IOException HttpGet request = new HttpGet(requestUrl); BasicCredentialsProvider provider = new BasicCredentialsProvider(); - provider.setCredentials( - new AuthScope(null, -1), - new UsernamePasswordCredentials(username, password.toCharArray())); + provider.setCredentials(new AuthScope(null, -1), new UsernamePasswordCredentials(username, password.toCharArray())); try (CloseableHttpClient httpClient = HttpClients.custom().setDefaultCredentialsProvider(provider).build()) { return httpClient.execute(request, new BasicHttpClientResponseHandler()); } From 050cf8222aed04ac2d9fc5dd7fa0e23dfa0502d9 Mon Sep 17 00:00:00 2001 From: laura Date: Tue, 29 Aug 2023 09:36:31 +0200 Subject: [PATCH 27/29] fixed failed build --- .../diagramrecognition/informants/ObjectDetectionInformant.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stages/diagram-recognition/src/main/kotlin/edu/kit/kastel/mcse/ardoco/lissa/diagramrecognition/informants/ObjectDetectionInformant.kt b/stages/diagram-recognition/src/main/kotlin/edu/kit/kastel/mcse/ardoco/lissa/diagramrecognition/informants/ObjectDetectionInformant.kt index 4a7dade90..dc2cf62e8 100644 --- a/stages/diagram-recognition/src/main/kotlin/edu/kit/kastel/mcse/ardoco/lissa/diagramrecognition/informants/ObjectDetectionInformant.kt +++ b/stages/diagram-recognition/src/main/kotlin/edu/kit/kastel/mcse/ardoco/lissa/diagramrecognition/informants/ObjectDetectionInformant.kt @@ -51,6 +51,6 @@ class ObjectDetectionInformant(dataRepository: DataRepository) : ImageProcessing val uploadFile = HttpPost("http://${hostIP()}:${container.apiPort}/sketches/") val multipart: HttpEntity = builder.build() uploadFile.entity = multipart - return executeRequest(uploadFile) + return executeRequest(uploadFile, true) } } From 6ea791da5c9c28b91e802ee693b2a690701c5c4f Mon Sep 17 00:00:00 2001 From: Jan Keim Date: Tue, 29 Aug 2023 17:13:47 +0200 Subject: [PATCH 28/29] Update ConfigManager to Enum --- .../informants/corenlp/config/ConfigManager.java | 14 ++++---------- .../corenlp/textprocessor/MicroserviceChecker.java | 4 ++-- .../corenlp/textprocessor/TextProcessor.java | 4 ++-- .../textprocessor/TextProcessorService.java | 5 +++-- 4 files changed, 11 insertions(+), 16 deletions(-) diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java index 343506c94..3135eb6c6 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java @@ -11,11 +11,12 @@ /** * This Singleton manages access to the config file. */ -public class ConfigManager { +public enum ConfigManager { - Logger logger = LoggerFactory.getLogger(ConfigManager.class); + INSTANCE; + + private final Logger logger = LoggerFactory.getLogger(ConfigManager.class); - private static ConfigManager instance; private final Properties properties; private static final String FILE_PATH = "config.properties"; private static final String PROPERTY_MICROSERVICE_URL = "microserviceUrl"; @@ -42,13 +43,6 @@ private ConfigManager() { } } - public static ConfigManager getInstance() { - if (instance == null) { - instance = new ConfigManager(); - } - return instance; - } - public String getMicroserviceUrl() { return properties.getProperty(PROPERTY_MICROSERVICE_URL); } diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java index 5dd0f219b..a9ac9077f 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java @@ -15,11 +15,11 @@ private MicroserviceChecker() { /** * checks if the CoreNLP microservice is available and can provide its services. - * + * * @return whether the microservice is available */ public static boolean isMicroserviceAvailable() throws IOException { - String requestUrl = ConfigManager.getInstance().getMicroserviceUrl() + ConfigManager.getInstance().getHealthService(); + String requestUrl = ConfigManager.INSTANCE.getMicroserviceUrl() + ConfigManager.INSTANCE.getHealthService(); String response = new HttpCommunicator().sendAuthenticatedGetRequest(requestUrl); return response.equals("Microservice is healthy"); } diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java index e3b6a28d6..ce523fc3c 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java @@ -21,7 +21,7 @@ public class TextProcessor { /** * processes and annotates a given text - * + * * @param inputText the input text * @return the annotated text */ @@ -33,7 +33,7 @@ public Text processText(String inputText) { microserviceAvailable = false; logger.warn("Could not check if CoreNLP microservice is available. ", e); } - if (ConfigManager.getInstance().getNlpProviderSource().equals("microservice") && microserviceAvailable) { + if (ConfigManager.INSTANCE.getNlpProviderSource().equals("microservice") && microserviceAvailable) { int numberOfTry = 0; while (numberOfTry < MAX_FAILED_SERVICE_REQUESTS) { try { diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java index 89e0d91a0..fae52259b 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java @@ -20,7 +20,7 @@ public class TextProcessorService { /** * processes and annotates a given text by sending requests to a microservice - * + * * @param inputText the input text * @return the annotated text */ @@ -33,7 +33,8 @@ public Text processText(String inputText) throws IOException, InvalidJsonExcepti private String sendCorenlpRequest(String inputText) throws IOException { inputText = URLEncoder.encode(inputText, StandardCharsets.UTF_8); - String requestUrl = ConfigManager.getInstance().getMicroserviceUrl() + ConfigManager.getInstance().getCorenlpService() + inputText; + ConfigManager configManager = ConfigManager.INSTANCE; + String requestUrl = configManager.getMicroserviceUrl() + configManager.getCorenlpService() + inputText; return sendAuthenticatedGetRequest(requestUrl); } From 246f43282fa2d7b3ffdfb98b0bdab9be557dc442 Mon Sep 17 00:00:00 2001 From: Jan Keim Date: Tue, 29 Aug 2023 17:24:25 +0200 Subject: [PATCH 29/29] Adapt MicroserviceChecker The checker now checks for exceptions explicitly --- .../corenlp/textprocessor/MicroserviceChecker.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java index a9ac9077f..4b7d89083 100644 --- a/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java +++ b/stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java @@ -20,8 +20,12 @@ private MicroserviceChecker() { */ public static boolean isMicroserviceAvailable() throws IOException { String requestUrl = ConfigManager.INSTANCE.getMicroserviceUrl() + ConfigManager.INSTANCE.getHealthService(); - String response = new HttpCommunicator().sendAuthenticatedGetRequest(requestUrl); - return response.equals("Microservice is healthy"); + try { + String response = new HttpCommunicator().sendAuthenticatedGetRequest(requestUrl); + return response.equals("Microservice is healthy"); + } catch (IOException e) { + return false; + } } }