Add corenlp provider microservice

stages/text-preprocessing/pom.xml

@@ -37,6 +37,11 @@
       <version>${stanford.corenlp.version}</version>
       <classifier>models</classifier>
     </dependency>
+    <dependency>
+      <groupId>io.github.ardoco</groupId>
+      <artifactId>text-provider-json</artifactId>
+      <version>0.8.0</version>
+    </dependency>
     <dependency>
       <groupId>io.github.ardoco.core</groupId>
       <artifactId>common</artifactId>
@@ -57,5 +62,10 @@
       <artifactId>slf4j-simple</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpclient</artifactId>
+        <version>4.5.13</version>
+    </dependency>
   </dependencies>
 </project>

stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/CoreNLPProvider.java

@@ -1,20 +1,16 @@
 /* Licensed under MIT 2022-2023. */
 package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp;
 
-import java.util.Properties;
 import java.util.SortedMap;
 
 import edu.kit.kastel.mcse.ardoco.core.api.PreprocessingData;
 import edu.kit.kastel.mcse.ardoco.core.api.text.NlpInformant;
 import edu.kit.kastel.mcse.ardoco.core.api.text.Text;
 import edu.kit.kastel.mcse.ardoco.core.common.util.DataRepositoryHelper;
 import edu.kit.kastel.mcse.ardoco.core.data.DataRepository;
-import edu.stanford.nlp.pipeline.CoreDocument;
-import edu.stanford.nlp.pipeline.StanfordCoreNLP;
+import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor.TextProcessor;
 
 public class CoreNLPProvider extends NlpInformant {
-    private static final String ANNOTATORS = "tokenize,ssplit,pos,parse,depparse,lemma"; // further: ",ner,coref"
-    private static final String DEPENDENCIES_ANNOTATION = "EnhancedPlusPlusDependenciesAnnotation";
 
     private Text annotatedText;
 
@@ -56,26 +52,8 @@ public synchronized Text getAnnotatedText() {
         return annotatedText;
     }
 
-    private static Properties getStanfordProperties(Properties properties) {
-        if (properties == null) {
-            throw new IllegalArgumentException("Properties are null");
-        }
-        var allStanfordProperties = new Properties(properties);
-        allStanfordProperties.setProperty("annotators", ANNOTATORS);
-
-        allStanfordProperties.put("parse", DEPENDENCIES_ANNOTATION);
-        allStanfordProperties.put("depparse", DEPENDENCIES_ANNOTATION);
-        allStanfordProperties.put("coref.algorithm", "fastneural");
-
-        return allStanfordProperties;
-    }
-
     private Text processText(String inputText) {
-        Properties props = getStanfordProperties(new Properties());
-        StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
-        CoreDocument document = new CoreDocument(inputText);
-        pipeline.annotate(document);
-        return new TextImpl(document);
+        return new TextProcessor().processText(inputText);
     }
 
     @Override

stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/TextImpl.java

@@ -10,13 +10,13 @@
 import edu.kit.kastel.mcse.ardoco.core.api.text.Word;
 import edu.stanford.nlp.pipeline.CoreDocument;
 
-class TextImpl implements Text {
+public class TextImpl implements Text {
 
     final CoreDocument coreDocument;
     private ImmutableList<Sentence> sentences = Lists.immutable.empty();
     private ImmutableList<Word> words = Lists.immutable.empty();
 
-    TextImpl(CoreDocument coreDocument) {
+    public TextImpl(CoreDocument coreDocument) {
         this.coreDocument = coreDocument;
     }
 

stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/config/ConfigManager.java

@@ -0,0 +1,68 @@
+/* Licensed under MIT 2023. */
+package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This Singleton manages access to the config file.
+ */
+public class ConfigManager {
+
+    Logger logger = LoggerFactory.getLogger(ConfigManager.class);
+
+    private static ConfigManager instance;
+    private final Properties properties;
+    private static final String FILE_PATH = "config.properties";
+    private static final String PROPERTY_MICROSERVICE_URL = "microserviceUrl";
+    private static final String PROPERTY_NLP_PROVIDER_SOURCE = "nlpProviderSource";
+    private static final String PROPERTY_CORENLP_SERVICE = "corenlpService";
+    private static final String PROPERTY_HEALTH_SERVICE = "healthService";
+
+    private ConfigManager() {
+        properties = new Properties();
+        try (InputStream fileInputStream = ConfigManager.class.getClassLoader().getResourceAsStream(FILE_PATH);) {
+            properties.load(fileInputStream);
+        } catch (IOException e) {
+            logger.warn("Could not load config file. ", e);
+            properties.setProperty(PROPERTY_MICROSERVICE_URL, "http://localhost:8080");
+            properties.setProperty(PROPERTY_NLP_PROVIDER_SOURCE, "local");
+            properties.setProperty(PROPERTY_CORENLP_SERVICE, "/stanfordnlp?text=");
+            properties.setProperty(PROPERTY_HEALTH_SERVICE, "/stanfordnlp/health");
+        }
+        if (System.getenv("MICROSERVICE_URL") != null) {
+            properties.setProperty(PROPERTY_MICROSERVICE_URL, System.getenv("MICROSERVICE_URL"));
+        }
+        if (System.getenv("NLP_PROVIDER_SOURCE") != null) {
+            properties.setProperty(PROPERTY_NLP_PROVIDER_SOURCE, System.getenv("NLP_PROVIDER_SOURCE"));
+        }
+    }
+
+    public static ConfigManager getInstance() {
+        if (instance == null) {
+            instance = new ConfigManager();
+        }
+        return instance;
+    }
+
+    public String getMicroserviceUrl() {
+        return properties.getProperty(PROPERTY_MICROSERVICE_URL);
+    }
+
+    public String getNlpProviderSource() {
+        return properties.getProperty(PROPERTY_NLP_PROVIDER_SOURCE);
+    }
+
+    public String getCorenlpService() {
+        return properties.getProperty(PROPERTY_CORENLP_SERVICE);
+    }
+
+    public String getHealthService() {
+        return properties.getProperty(PROPERTY_HEALTH_SERVICE);
+    }
+
+}

stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/HttpCommunicator.java

@@ -0,0 +1,60 @@
+package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+
+public class HttpCommunicator {
+
+    /**
+     * The returned CloseableHttpResponse must be closed after usage.
+     * @param requestUrl    The URL to send the GET request to.
+     * @return              The response of the GET request.
+     * @throws IOException   If the request fails.
+     */
+    public CloseableHttpResponse sendAuthenticatedGetRequest(String requestUrl) throws IOException {
+        String username = System.getenv("USERNAME");
+        String password = System.getenv("PASSWORD");
+        if (username == null || password == null) {
+            throw new IOException("Environment variables USERNAME and PASSWORD must be set.");
+        }
+
+        HttpGet request = new HttpGet(requestUrl);
+        CredentialsProvider provider = new BasicCredentialsProvider();
+        provider.setCredentials(
+                AuthScope.ANY,
+                new UsernamePasswordCredentials(username, password)
+        );
+
+        CloseableHttpClient httpClient = HttpClientBuilder.create()
+                .setDefaultCredentialsProvider(provider)
+                .build();
+        return httpClient.execute(request);
+    }
+
+    public String readGetResponse(CloseableHttpResponse response) throws IOException {
+        if (response.getStatusLine().getStatusCode() != HttpURLConnection.HTTP_OK) {
+            throw new IOException("HTTP error code: " + response.getStatusLine().getStatusCode());
+        }
+        HttpEntity entity = response.getEntity();
+        BufferedReader in = new BufferedReader(new InputStreamReader(entity.getContent()));
+        String inputLine;
+        StringBuilder content = new StringBuilder();
+        while ((inputLine = in.readLine()) != null) {
+            content.append(inputLine);
+        }
+        in.close();
+        return content.toString();
+    }
+}

stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/MicroserviceChecker.java

@@ -0,0 +1,31 @@
+/* Licensed under MIT 2023. */
+package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+
+import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config.ConfigManager;
+
+import org.apache.http.client.methods.CloseableHttpResponse;
+
+/**
+ * This utility class provides methods to check whether the microservice is available.
+ */
+public final class MicroserviceChecker {
+
+    private MicroserviceChecker() {
+    }
+
+    /**
+     * checks if the CoreNLP microservice is available and can provide its services.
+     * 
+     * @return whether the microservice is available
+     */
+    public static boolean isMicroserviceAvailable() throws IOException {
+        String requestUrl = ConfigManager.getInstance().getMicroserviceUrl() + ConfigManager.getInstance().getHealthService();
+        CloseableHttpResponse response = new HttpCommunicator().sendAuthenticatedGetRequest(requestUrl);
+        response.close();
+        return response.getStatusLine().getStatusCode() == HttpURLConnection.HTTP_OK;
+    }
+
+}

stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessor.java

@@ -0,0 +1,65 @@
+/* Licensed under MIT 2023. */
+package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import edu.kit.kastel.mcse.ardoco.core.api.text.Text;
+import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.config.ConfigManager;
+import io.github.ardoco.textproviderjson.error.InvalidJsonException;
+import io.github.ardoco.textproviderjson.error.NotConvertableException;
+
+/**
+ * This text processor processes texts using CoreNLP.
+ */
+public class TextProcessor {
+
+    private static final int MAX_FAILED_SERVICE_REQUESTS = 2;
+    Logger logger = LoggerFactory.getLogger(TextProcessor.class);
+
+    /**
+     * processes and annotates a given text
+     * 
+     * @param inputText the input text
+     * @return the annotated text
+     */
+    public Text processText(String inputText) {
+        boolean microserviceAvailable;
+        try {
+            microserviceAvailable = MicroserviceChecker.isMicroserviceAvailable();
+        } catch (IOException e) {
+            microserviceAvailable = false;
+            logger.warn("Could not check if CoreNLP microservice is available. ", e);
+        }
+        if (ConfigManager.getInstance().getNlpProviderSource().equals("microservice") && microserviceAvailable) {
+            int numberOfTry = 0;
+            while (numberOfTry < MAX_FAILED_SERVICE_REQUESTS) {
+                try {
+                    Text processedText = processService(inputText);
+                    logger.info("Processed text with CoreNLP microservice.");
+                    return processedText;
+                } catch (IOException e) {
+                    numberOfTry++;
+                    logger.warn("Could not process text with CoreNLP microservice. Trying again. ", e);
+                } catch (NotConvertableException | InvalidJsonException e) {
+                    logger.warn("Could not process text with CoreNLP microservice. Text not convertable. ", e);
+                    return processLocally(inputText);
+                }
+            }
+            logger.warn("Could not process text with CoreNLP microservice. Processing locally instead.");
+        }
+        logger.info("Processed text locally.");
+        return processLocally(inputText);
+    }
+
+    private Text processLocally(String inputText) {
+        return new TextProcessorLocal().processText(inputText);
+    }
+
+    private Text processService(String inputText) throws IOException, NotConvertableException, InvalidJsonException {
+        return new TextProcessorService().processText(inputText);
+    }
+
+}

stages/text-preprocessing/src/main/java/edu/kit/kastel/mcse/ardoco/core/text/providers/informants/corenlp/textprocessor/TextProcessorLocal.java

@@ -0,0 +1,45 @@
+/* Licensed under MIT 2023. */
+package edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.textprocessor;
+
+import java.util.Properties;
+
+import edu.kit.kastel.mcse.ardoco.core.api.text.Text;
+import edu.kit.kastel.mcse.ardoco.core.text.providers.informants.corenlp.TextImpl;
+import edu.stanford.nlp.pipeline.CoreDocument;
+import edu.stanford.nlp.pipeline.StanfordCoreNLP;
+
+/**
+ * This text processor processes texts locally using CoreNLP.
+ */
+public class TextProcessorLocal {
+    private static final String ANNOTATORS = "tokenize,ssplit,pos,parse,depparse,lemma"; // further: ",ner,coref"
+    private static final String DEPENDENCIES_ANNOTATION = "EnhancedPlusPlusDependenciesAnnotation";
+
+    /**
+     * processes and annotates a given text locally using CoreNLP.
+     *
+     * @param inputText the input text
+     * @return the annotated text
+     */
+    public Text processText(String inputText) {
+        Properties props = getStanfordProperties(new Properties());
+        StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
+        CoreDocument document = new CoreDocument(inputText);
+        pipeline.annotate(document);
+        return new TextImpl(document);
+    }
+
+    private static Properties getStanfordProperties(Properties properties) {
+        if (properties == null) {
+            throw new IllegalArgumentException("Properties are null");
+        }
+        var allStanfordProperties = new Properties(properties);
+        allStanfordProperties.setProperty("annotators", ANNOTATORS);
+
+        allStanfordProperties.put("parse", DEPENDENCIES_ANNOTATION);
+        allStanfordProperties.put("depparse", DEPENDENCIES_ANNOTATION);
+        allStanfordProperties.put("coref.algorithm", "fastneural");
+
+        return allStanfordProperties;
+    }
+}