Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature limit resources #561

Merged
merged 25 commits into from
Dec 18, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
4b857f9
Increased the version of the core library and updated the code accord…
MichaelRoeder Sep 11, 2023
feced4c
Increased commons-io version.
MichaelRoeder Sep 11, 2023
3ad6b20
Implemented the hardware constraint processing. Fixed several compile…
MichaelRoeder Oct 14, 2023
f082866
Implemented the hardware constraint processing. Fixed several compile…
MichaelRoeder Oct 14, 2023
e3d7034
Implemented the processing of hardware constraints. Fixed some smalle…
MichaelRoeder Oct 14, 2023
62c9346
Merged changes from develop.
MichaelRoeder Nov 4, 2023
8c97198
Adapted file persmissions.
MichaelRoeder Nov 11, 2023
37cfda3
Fixed issues with the hardware constraints.
MichaelRoeder Nov 11, 2023
5402818
Merge branch 'develop' into feature-limit-resources
MichaelRoeder Nov 11, 2023
98c9d66
Moved the web.xml used in cases where no authentication is needed int…
MichaelRoeder Nov 13, 2023
c9da48f
Added the missing configure-auth script.
MichaelRoeder Nov 13, 2023
2823711
Fixed pom version problem.
MichaelRoeder Nov 17, 2023
d3cb4ba
Removed user information that is not needed for retrieving the licens…
MichaelRoeder Nov 17, 2023
18614df
Fixed pom version problem.
MichaelRoeder Nov 17, 2023
627b58c
Added the handling of error reports. Errors are added to the result m…
MichaelRoeder Dec 8, 2023
1eb5c71
Included the new blocking rabbitmq receiver in the storage service.
MichaelRoeder Dec 14, 2023
7cbb4e7
Included the new blocking rabbitmq receiver in the analysis component.
MichaelRoeder Dec 14, 2023
6803d50
Merge branch 'feature/build-in-docker' into feature-limit-resources
MichaelRoeder Dec 14, 2023
5d5eae9
Updated dev docker-compose.
MichaelRoeder Dec 15, 2023
861895f
Fixed problems with the queue-based consumer.
MichaelRoeder Dec 16, 2023
2e630e5
Finished implementation of error report handling.
MichaelRoeder Dec 16, 2023
1c5c314
Removed the errorDetails property since it is not used anymore.
MichaelRoeder Dec 17, 2023
25ab24d
Prepared release of 2.0.17.
MichaelRoeder Dec 17, 2023
f173e7f
Removed faulty file.
MichaelRoeder Dec 18, 2023
ff5d0c1
Applied changes according to review.
MichaelRoeder Dec 18, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion analysis-component/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
<parent>
<groupId>org.hobbit</groupId>
<artifactId>parent</artifactId>
<version>${hobbitplatform.version}</version>
<version>2.0.17</version>
<relativePath>../parent-pom</relativePath>
</parent>
<artifactId>analysis-component</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,22 +16,42 @@
*/
package org.hobbit.analysis;

import java.util.stream.Stream;
import java.util.stream.Collectors;
import java.io.IOException;
import java.util.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import org.aksw.palmetto.evaluate.correlation.PearsonsSampleCorrelationCoefficient;
import org.apache.commons.io.IOUtils;
import org.apache.jena.datatypes.RDFDatatype;
import org.apache.jena.datatypes.xsd.XSDDatatype;
import org.apache.jena.rdf.model.*;
import org.apache.jena.rdf.model.Model;
import org.apache.jena.rdf.model.ModelFactory;
import org.apache.jena.rdf.model.Property;
import org.apache.jena.rdf.model.RDFNode;
import org.apache.jena.rdf.model.ResIterator;
import org.apache.jena.rdf.model.Resource;
import org.apache.jena.rdf.model.ResourceFactory;
import org.apache.jena.rdf.model.Statement;
import org.apache.jena.rdf.model.StmtIterator;
import org.apache.jena.vocabulary.DCTerms;
import org.apache.jena.vocabulary.RDF;
import org.apache.jena.vocabulary.XSD;
import org.apache.commons.io.IOUtils;
import org.hobbit.core.Constants;
import org.hobbit.core.components.AbstractComponent;
import org.hobbit.core.data.RabbitQueue;
import org.hobbit.core.rabbit.QueueingConsumer;
import org.hobbit.core.rabbit.RabbitMQUtils;
import org.hobbit.storage.client.StorageServiceClient;
import org.hobbit.storage.queries.SparqlQueries;
Expand All @@ -42,20 +62,31 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.rabbitmq.client.QueueingConsumer;
import com.rabbitmq.client.Delivery;

import weka.attributeSelection.*;
import weka.attributeSelection.AttributeSelection;
import weka.attributeSelection.CfsSubsetEval;
import weka.attributeSelection.GreedyStepwise;
import weka.classifiers.functions.LinearRegression;
import weka.clusterers.SimpleKMeans;
import weka.core.*;
import weka.core.Attribute;
import weka.core.DenseInstance;
import weka.core.Instance;
import weka.core.Instances;
import weka.core.SparseInstance;


/**
* This class implements the functionality for the Analysis Component
* TODO:: !!REFACTOR INTO A MORE GENERIC DESIGN!!
*/
public class AnalysisComponent extends AbstractComponent {
/**
* The time the main receiver thread will sleep if it didn't receive any message
* (in seconds).
*/
private static final int WAITING_TIME_BEFORE_CHECKING_STATUS = 60;

private static final Logger LOGGER = LoggerFactory.getLogger(AnalysisComponent.class);
private static final String GRAPH_URI = Constants.PUBLIC_RESULT_GRAPH_URI;
protected RabbitQueue controller2AnalysisQueue;
Expand All @@ -82,16 +113,24 @@ public void init() throws Exception {
@Override
public void run() throws Exception {
LOGGER.info("Awaiting requests");
QueueingConsumer.Delivery delivery;
Delivery delivery;
while (true) {
delivery = consumer.nextDelivery();
delivery = null;
// Let's wait for a delivery for 60 seconds
try {
delivery = consumer.getDeliveryQueue().poll(WAITING_TIME_BEFORE_CHECKING_STATUS, TimeUnit.SECONDS);
} catch (InterruptedException e) {
// interrupted; just continue
}
if (delivery != null) {
LOGGER.info("Received a request. Processing...");
String expUri = RabbitMQUtils.readString(delivery.getBody());
handleRequest(expUri);
} else {
// This would be the place at which we could react to signals, e.g., terminate
// the service if needed.
}
}

}

protected void handleRequest(String expUri) {
Expand Down Expand Up @@ -235,18 +274,17 @@ public void close() throws IOException {
* The model of the experiment
* @return Returns the analyzed model
*/

/**
/*
private AnalysisModel analyseExperiment(Model experimentModel, String expUri){
AnalysisModel analysisModel = new AnalysisModel(experimentModel, expUri);
analysisModel.analyse();
return analysisModel;
}*/

private void notifyQueue(){
/*private void notifyQueue(){
//TODO:: return the status of component

}
}*/

/**
* This class implements the functionality of the Analysis Model which
Expand Down
18 changes: 11 additions & 7 deletions docker-compose-dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,20 @@ services:
HOBBIT_RABBIT_IMAGE: "rabbitmq:management"
HOBBIT_RABBIT_HOST: "rabbit"
HOBBIT_REDIS_HOST: "redis"
DEPLOY_ENV: "testing"
DEPLOY_ENV: "develop"
GITLAB_USER: "${GITLAB_USER}"
GITLAB_EMAIL: "${GITLAB_EMAIL}"
GITLAB_TOKEN: "${GITLAB_TOKEN}"
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did this cause any problems? Having variables here by default makes it possible to just add .env file without making any additional changes.

LOGGING_GELF_ADDRESS: "udp://localhost:12201"
#LOGGING_GELF_ADDRESS: "udp://localhost:12201"
SWARM_NODE_NUMBER: "1"
PROMETHEUS_HOST: prometheus
PROMETHEUS_PORT: 9090
USE_GITLAB: "false"
LOCAL_METADATA_DIRECTORY: "/metadata"
DOCKER_AUTOPULL: "0"
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- ./metadata:/metadata

# HOBBIT GUI
gui:
Expand All @@ -37,8 +41,7 @@ services:
- ELASTICSEARCH_HOST=elasticsearch
- ELASTICSEARCH_HTTP_PORT=9200
- USE_UI_AUTH=false
volumes:
- ./config/jetty/web-without-ui-auth.xml:/var/lib/jetty/webapps/ROOT/WEB-INF/web.xml
#volumes:
#- /data/docker/messages/global.html:/var/lib/jetty/webapps/messages/global.html
#- /data/docker/messages/benchmark.html:/var/lib/jetty/webapps/messages/benchmark.html
#- /data/docker/messages/status.html:/var/lib/jetty/webapps/messages/status.html
Expand Down Expand Up @@ -92,7 +95,8 @@ services:
stop_signal: SIGINT
stop_grace_period: 2m
volumes:
- ./config/db:/opt/virtuoso-opensource/database
- ./config/db:/opt/virtuoso-opensource/var/lib/virtuoso/db
#- ./config/db:/opt/virtuoso-opensource/database
networks:
- hobbit-core
ports:
Expand All @@ -106,8 +110,8 @@ services:
environment:
- SPARQL_ENDPOINT_URL=http://vos:8890/sparql
- HOBBIT_RABBIT_HOST=rabbit
- SPARQL_ENDPOINT_USERNAME=HobbitPlatform
- SPARQL_ENDPOINT_PASSWORD=Password
- SPARQL_ENDPOINT_USERNAME=dba
- SPARQL_ENDPOINT_PASSWORD=dba

node-exporter:
image: prom/node-exporter
Expand Down
13 changes: 10 additions & 3 deletions hobbit-gui/gui-serverbackend/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,15 @@
# https://www.npmjs.com/package/node-sass#node-version-support-policy
# node-sass 4.10.0 (../gui-client/package-lock.json) ⇒ node <= 11

# Build web pages
FROM node:11 AS build-client
WORKDIR /usr/src/hobbit-platform
COPY hobbit-gui/gui-client/package*.json ./hobbit-gui/gui-client/
RUN npm --prefix hobbit-gui/gui-client ci --omit=dev
COPY hobbit-gui/gui-client ./hobbit-gui/gui-client
RUN npm --prefix hobbit-gui/gui-client run build-prod

# Build Java backend
FROM maven:3-eclipse-temurin-11 AS build
WORKDIR /usr/src/hobbit-platform
COPY parent-pom/pom.xml ./parent-pom/
Expand All @@ -20,14 +22,19 @@ COPY hobbit-gui/gui-serverbackend/src ./hobbit-gui/gui-serverbackend/src
COPY --from=build-client /usr/src/hobbit-platform/hobbit-gui/gui-client/dist ./hobbit-gui/gui-client/dist
RUN mvn --file ./hobbit-gui/gui-serverbackend/ -Dmaven.test.skip=true package

# Set up web server
FROM jetty:9.3-jre8

# Add script to register keycloak
COPY hobbit-gui/gui-serverbackend/configure-auth.sh $JETTY_BASE/configure-auth.sh
# Download the keycloak adapter and add it
RUN cd $JETTY_BASE && \
curl -L -O http://hobbitdata.informatik.uni-leipzig.de/hobbit/keycloak-jetty93-adapter-for-hobbit-dist-2.4.0.Final.zip && \
unzip keycloak-jetty93-adapter-for-hobbit-dist-2.4.0.Final.zip && \
rm -f keycloak-jetty93-adapter-for-hobbit-dist-2.4.0.Final.zip && \
java -jar $JETTY_HOME/start.jar --add-to-startd=keycloak

# Copy message definitions
COPY hobbit-gui/gui-serverbackend/messages /var/lib/jetty/webapps/messages
# Copy Java backend
COPY --chown=jetty --chmod=777 --from=build /usr/src/hobbit-platform/hobbit-gui/gui-serverbackend/target/gui-serverbackend $JETTY_BASE/webapps/ROOT

COPY --from=build /usr/src/hobbit-platform/hobbit-gui/gui-serverbackend/target/gui-serverbackend $JETTY_BASE/webapps/ROOT
ENTRYPOINT ./configure-auth.sh && /docker-entrypoint.sh
9 changes: 9 additions & 0 deletions hobbit-gui/gui-serverbackend/configure-auth.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
#!/bin/bash
# Runs the GUI serverbackend of the HOBBIT Platform.
USE_KEYCLOAK="${USE_UI_AUTH:-true}"

if [ "${USE_KEYCLOAK}" == "false" ]; then
echo "Replacing web.xml with web-without-ui-auth.xml"
cp /var/lib/jetty/webapps/ROOT/WEB-INF/web-without-ui-auth.xml /var/lib/jetty/webapps/ROOT/WEB-INF/web.xml
fi

2 changes: 1 addition & 1 deletion hobbit-gui/gui-serverbackend/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
<parent>
<groupId>org.hobbit</groupId>
<artifactId>parent</artifactId>
<version>${hobbitplatform.version}</version>
<version>2.0.17</version>
<relativePath>../../parent-pom</relativePath>
</parent>
<artifactId>gui-serverbackend</artifactId>
Expand Down
Loading
Loading