Add corenlp provider microservice

stages/text-preprocessing/pom.xml

@@ -37,6 +37,11 @@
       <version>${stanford.corenlp.version}</version>
       <classifier>models</classifier>
     </dependency>
+    <dependency>
+      <groupId>io.github.ardoco</groupId>
+      <artifactId>text-provider-json</artifactId>
+      <version>0.8.0</version>
+    </dependency>
     <dependency>
       <groupId>io.github.ardoco.core</groupId>
       <artifactId>common</artifactId>

stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/CoreNLPProvider.java

@@ -2,19 +2,15 @@
 package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp;
 
 import java.util.Map;
-import java.util.Properties;
 
 import edu.kit.kastel.mcse.ardoco.core.api.PreprocessingData;
 import edu.kit.kastel.mcse.ardoco.core.api.text.NlpInformant;
 import edu.kit.kastel.mcse.ardoco.core.api.text.Text;
 import edu.kit.kastel.mcse.ardoco.core.common.util.DataRepositoryHelper;
 import edu.kit.kastel.mcse.ardoco.core.data.DataRepository;
-import edu.stanford.nlp.pipeline.CoreDocument;
-import edu.stanford.nlp.pipeline.StanfordCoreNLP;
+import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor.TextProcessor;
 
 public class CoreNLPProvider extends NlpInformant {
-    private static final String ANNOTATORS = "tokenize,ssplit,pos,parse,depparse,lemma"; // further: ",ner,coref"
-    private static final String DEPENDENCIES_ANNOTATION = "EnhancedPlusPlusDependenciesAnnotation";
 
     private Text annotatedText;
 
@@ -56,26 +52,8 @@ public synchronized Text getAnnotatedText() {
         return annotatedText;
     }
 
-    private static Properties getStanfordProperties(Properties properties) {
-        if (properties == null) {
-            throw new IllegalArgumentException("Properties are null");
-        }
-        var allStanfordProperties = new Properties(properties);
-        allStanfordProperties.setProperty("annotators", ANNOTATORS);
-
-        allStanfordProperties.put("parse", DEPENDENCIES_ANNOTATION);
-        allStanfordProperties.put("depparse", DEPENDENCIES_ANNOTATION);
-        allStanfordProperties.put("coref.algorithm", "fastneural");
-
-        return allStanfordProperties;
-    }
-
     private Text processText(String inputText) {
-        Properties props = getStanfordProperties(new Properties());
-        StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
-        CoreDocument document = new CoreDocument(inputText);
-        pipeline.annotate(document);
-        return new TextImpl(document);
+        return new TextProcessor().processText(inputText);
     }
 
     @Override

stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextImpl.java

@@ -10,13 +10,13 @@
 import edu.kit.kastel.mcse.ardoco.core.api.text.Word;
 import edu.stanford.nlp.pipeline.CoreDocument;
 
-class TextImpl implements Text {
+public class TextImpl implements Text {
 
     final CoreDocument coreDocument;
     private ImmutableList<Sentence> sentences = Lists.immutable.empty();
     private ImmutableList<Word> words = Lists.immutable.empty();
 
-    TextImpl(CoreDocument coreDocument) {
+    public TextImpl(CoreDocument coreDocument) {
         this.coreDocument = coreDocument;
     }
 

stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java

@@ -0,0 +1,67 @@
+/* Licensed under MIT 2023. */
+package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This Singleton manages access to the config file.
+ */
+public class ConfigManager {
+
+    Logger logger = LoggerFactory.getLogger(ConfigManager.class);
+
+    private static ConfigManager instance;
+    private final Properties properties;
+    private static final String FILE_PATH = "config.properties";
+
+    private ConfigManager() {
+        properties = new Properties();
+        try (InputStream fileInputStream = ConfigManager.class.getClassLoader().getResourceAsStream(FILE_PATH);) {
+            properties.load(fileInputStream);
+        } catch (IOException e) {
+            logger.warn("Could not load config file. ", e);
+            properties.setProperty("microserviceUrl", "http://localhost:8080");
+            properties.setProperty("nlpProviderSource", "local");
+            properties.setProperty("corenlpService", "/stanfordnlp?text=");
+            properties.setProperty("healthService", "/stanfordnlp/health");
+        }
+        if (System.getenv("MICROSERVICE_URL") != null) {
+            properties.setProperty("microserviceUrl", System.getenv("MICROSERVICE_URL"));
+        }
+        if (System.getenv("NLP_PROVIDER_SOURCE") != null) {
+            properties.setProperty("nlpProviderSource", System.getenv("NLP_PROVIDER_SOURCE"));
+        }
+    }
+
+    public static ConfigManager getInstance() {
+        if (instance == null) {
+            instance = new ConfigManager();
+        }
+        return instance;
+    }
+
+    /**
+     * gets the value of the given key in the config file
+     * 
+     * @param key the key
+     * @return the value
+     */
+    public String getProperty(String key) {
+        return properties.getProperty(key);
+    }
+
+    /**
+     * sets the value of the given key in the config file
+     * 
+     * @param key   the key
+     * @param value the new value
+     */
+    public void setProperty(String key, String value) {
+        properties.setProperty(key, value);
+    }
+}

stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java

@@ -0,0 +1,47 @@
+/* Licensed under MIT 2023. */
+package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.Base64;
+
+import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config.ConfigManager;
+
+/**
+ * This utility class provides methods to check whether the microservice is available.
+ */
+public final class MicroserviceChecker {
+
+    private MicroserviceChecker() {
+    }
+
+    /**
+     * checks if the CoreNLP microservice is available and can provide its services.
+     * 
+     * @return whether the microservice is available
+     */
+    public static boolean isMicroserviceAvailable() throws IOException {
+        String requestUrl = ConfigManager.getInstance().getProperty("microserviceUrl") + ConfigManager.getInstance().getProperty("healthService");
+
+        String username = System.getenv("USERNAME");
+        String password = System.getenv("PASSWORD");
+        if (username == null || password == null) {
+            throw new IOException("Environment variables USERNAME and PASSWORD must be set.");
+        }
+        URL url = new URL(requestUrl);
+        HttpURLConnection con = (HttpURLConnection) url.openConnection();
+        con.setRequestMethod("GET");
+        con.setConnectTimeout(5000); // timeout after 5 sec
+
+        // Encode the username and password
+        String authString = username + ":" + password;
+        String encodedAuthString = Base64.getEncoder().encodeToString(authString.getBytes());
+        String authHeaderValue = "Basic " + encodedAuthString;
+        con.setRequestProperty("Authorization", authHeaderValue);
+        int statusCode = con.getResponseCode();
+        con.disconnect();
+        return statusCode == HttpURLConnection.HTTP_OK;
+    }
+
+}

stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java

@@ -0,0 +1,65 @@
+/* Licensed under MIT 2023. */
+package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import edu.kit.kastel.mcse.ardoco.core.api.text.Text;
+import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config.ConfigManager;
+import io.github.ardoco.textproviderjson.error.InvalidJsonException;
+import io.github.ardoco.textproviderjson.error.NotConvertableException;
+
+/**
+ * This text processor processes texts using CoreNLP.
+ */
+public class TextProcessor {
+
+    private static final int MAX_FAILED_SERVICE_REQUESTS = 2;
+    Logger logger = LoggerFactory.getLogger(TextProcessor.class);
+
+    /**
+     * processes and annotates a given text
+     * 
+     * @param inputText the input text
+     * @return the annotated text
+     */
+    public Text processText(String inputText) {
+        boolean microserviceAvailable;
+        try {
+            microserviceAvailable = MicroserviceChecker.isMicroserviceAvailable();
+        } catch (IOException e) {
+            microserviceAvailable = false;
+            logger.warn("Could not check if CoreNLP microservice is available. ", e);
+        }
+        if (ConfigManager.getInstance().getProperty("nlpProviderSource").equals("microservice") && microserviceAvailable) {
+            int k = 0;
+            while (k < MAX_FAILED_SERVICE_REQUESTS) {
+                try {
+                    Text processedText = processService(inputText);
+                    logger.info("Processed text with CoreNLP microservice.");
+                    return processedText;
+                } catch (IOException e) {
+                    k++;
+                    logger.warn("Could not process text with CoreNLP microservice. Trying again. ", e);
+                } catch (NotConvertableException | InvalidJsonException e) {
+                    logger.warn("Could not process text with CoreNLP microservice. Text not convertable. ", e);
+                    return processLocally(inputText);
+                }
+            }
+            logger.warn("Could not process text with CoreNLP microservice. Processing locally instead.");
+        }
+        logger.info("Processed text locally.");
+        return processLocally(inputText);
+    }
+
+    private Text processLocally(String inputText) {
+        return new TextProcessorLocal().processText(inputText);
+    }
+
+    private Text processService(String inputText) throws IOException, NotConvertableException, InvalidJsonException {
+        return new TextProcessorService().processText(inputText);
+    }
+
+}

stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorLocal.java

@@ -0,0 +1,45 @@
+/* Licensed under MIT 2023. */
+package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor;
+
+import java.util.Properties;
+
+import edu.kit.kastel.mcse.ardoco.core.api.text.Text;
+import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.TextImpl;
+import edu.stanford.nlp.pipeline.CoreDocument;
+import edu.stanford.nlp.pipeline.StanfordCoreNLP;
+
+/**
+ * This text processor processes texts locally using CoreNLP.
+ */
+public class TextProcessorLocal {
+    private static final String ANNOTATORS = "tokenize,ssplit,pos,parse,depparse,lemma"; // further: ",ner,coref"
+    private static final String DEPENDENCIES_ANNOTATION = "EnhancedPlusPlusDependenciesAnnotation";
+
+    /**
+     * processes and annotates a given text locally using CoreNLP.
+     *
+     * @param inputText the input text
+     * @return the annotated text
+     */
+    public Text processText(String inputText) {
+        Properties props = getStanfordProperties(new Properties());
+        StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
+        CoreDocument document = new CoreDocument(inputText);
+        pipeline.annotate(document);
+        return new TextImpl(document);
+    }
+
+    private static Properties getStanfordProperties(Properties properties) {
+        if (properties == null) {
+            throw new IllegalArgumentException("Properties are null");
+        }
+        var allStanfordProperties = new Properties(properties);
+        allStanfordProperties.setProperty("annotators", ANNOTATORS);
+
+        allStanfordProperties.put("parse", DEPENDENCIES_ANNOTATION);
+        allStanfordProperties.put("depparse", DEPENDENCIES_ANNOTATION);
+        allStanfordProperties.put("coref.algorithm", "fastneural");
+
+        return allStanfordProperties;
+    }
+}

stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorService.java

@@ -0,0 +1,79 @@
+/* Licensed under MIT 2023. */
+package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+
+import edu.kit.kastel.mcse.ardoco.core.api.text.Text;
+import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config.ConfigManager;
+import io.github.ardoco.textproviderjson.converter.DtoToObjectConverter;
+import io.github.ardoco.textproviderjson.converter.JsonConverter;
+import io.github.ardoco.textproviderjson.dto.TextDto;
+import io.github.ardoco.textproviderjson.error.InvalidJsonException;
+import io.github.ardoco.textproviderjson.error.NotConvertableException;
+
+/**
+ * This text processor processes texts by sending requests to a microservice, which provides text processing using CoreNLP.
+ */
+public class TextProcessorService {
+
+    /**
+     * processes and annotates a given text by sending requests to a microservice
+     * 
+     * @param inputText the input text
+     * @return the annotated text
+     */
+    public Text processText(String inputText) throws IOException, InvalidJsonException, NotConvertableException {
+        TextDto textDto;
+        String jsonText = sendCorenlpRequest(inputText);
+        textDto = JsonConverter.fromJsonString(jsonText);
+        return new DtoToObjectConverter().convertText(textDto);
+    }
+
+    private String sendCorenlpRequest(String inputText) throws IOException {
+        inputText = URLEncoder.encode(inputText, StandardCharsets.UTF_8);
+        String requestUrl = ConfigManager.getInstance().getProperty("microserviceUrl") + ConfigManager.getInstance().getProperty("corenlpService") + inputText;
+        return sendAuthenticatedGetRequest(requestUrl);
+    }
+
+    public String sendAuthenticatedGetRequest(String requestUrl) throws IOException {
+        String username = System.getenv("USERNAME");
+        String password = System.getenv("PASSWORD");
+        if (username == null || password == null) {
+            throw new IOException("Environment variables USERNAME and PASSWORD must be set.");
+        }
+        URL url = new URL(requestUrl);
+        HttpURLConnection con = (HttpURLConnection) url.openConnection();
+        con.setRequestMethod("GET");
+
+        // Encode the username and password
+        String authString = username + ":" + password;
+        String encodedAuthString = Base64.getEncoder().encodeToString(authString.getBytes());
+        String authHeaderValue = "Basic " + encodedAuthString;
+        con.setRequestProperty("Authorization", authHeaderValue);
+
+        String content = readGetResponse(con);
+        con.disconnect();
+        return content;
+    }
+
+    private String readGetResponse(HttpURLConnection con) throws IOException {
+        if (con.getResponseCode() != HttpURLConnection.HTTP_OK) {
+            throw new IOException("HTTP error code: " + con.getResponseCode());
+        }
+        BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()));
+        String inputLine;
+        StringBuilder content = new StringBuilder();
+        while ((inputLine = in.readLine()) != null) {
+            content.append(inputLine);
+        }
+        in.close();
+        return content.toString();
+    }
+}

stages/text-preprocessing/src/main/resources/config.properties

@@ -0,0 +1,5 @@
+nlpProviderSource=microservice
+microserviceUrl= http://localhost:8080
+corenlpService=/stanfordnlp?text=
+healthService=/stanfordnlp/health
+