Merge pull request #562 from hobbit-project/develop

Developed version 2.0.17

.github/workflows/make.yml

@@ -2,7 +2,8 @@ name: make
 on: push
 jobs:
   make:
-    runs-on: ubuntu-20.04
+    # https://docs.github.com/en/actions/using-jobs/choosing-the-runner-for-a-job
+    runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-java@v1
@@ -13,4 +14,5 @@ jobs:
         node-version: 8
     - run: |
         docker swarm init
+    - run: docker compose -f docker-compose-dev.yml pull cadvisor node-exporter prometheus rabbit redis
     - run: make test

Makefile

@@ -1,5 +1,4 @@
-# build platform components
-default: build
+build: build-dev-images
 
 deploy: create-networks start
 
@@ -26,36 +25,19 @@ start-dev-platform:
 start-dev-elk:
 	docker-compose -f docker-compose-elk.yml up -d
 
-build: build-java build-dev-images
-
-build-java: install-parent-pom build-controller build-storage build-analysis build-gui
-
-build-gui:
-	cd hobbit-gui/gui-client && sh -c 'test "$$TRAVIS" = "true" && npm ci; true' && sh -c 'test "$$TRAVIS" = "true" || npm install; true' && npm run build-prod
-	cd hobbit-gui/gui-serverbackend && mvn clean package
-
-build-controller:
-	cd platform-controller && make build
-
-build-storage:
-	cd platform-storage/storage-service && mvn clean package -U
-
-build-analysis:
-	cd analysis-component && mvn clean package -U
-
 build-dev-images: build-dev-platform-controller-image build-dev-gui-image build-dev-analysis-image build-dev-storage-image
 
 build-dev-platform-controller-image:
-	docker build -t hobbitproject/hobbit-platform-controller:dev ./platform-controller
+	docker build -t hobbitproject/hobbit-platform-controller:dev --file platform-controller/Dockerfile .
 
 build-dev-gui-image:
-	docker build -t hobbitproject/hobbit-gui:dev ./hobbit-gui/gui-serverbackend
+	docker build -t hobbitproject/hobbit-gui:dev --file hobbit-gui/gui-serverbackend/Dockerfile .
 
 build-dev-analysis-image:
-	docker build -t hobbitproject/hobbit-analysis-component:dev ./analysis-component
+	docker build -t hobbitproject/hobbit-analysis-component:dev --file ./analysis-component/Dockerfile .
 
 build-dev-storage-image:
-	docker build -t hobbitproject/hobbit-storage-service:dev ./platform-storage/storage-service
+	docker build -t hobbitproject/hobbit-storage-service:dev --file ./platform-storage/storage-service/Dockerfile .
 
 create-networks:
 	@docker network inspect hobbit >/dev/null || (docker network create -d overlay --attachable --subnet 172.16.100.0/24 hobbit && echo "Created network: hobbit")

README.md

@@ -1,4 +1,4 @@
-[![Build Status](https://travis-ci.org/hobbit-project/platform.svg?branch=master)](https://travis-ci.org/hobbit-project/platform)
+[![make](https://github.com/hobbit-project/platform/actions/workflows/make.yml/badge.svg?branch=master)](https://github.com/hobbit-project/platform/actions/workflows/make.yml)
 
 # HOBBIT platform
 

analysis-component/Dockerfile

@@ -1,5 +1,13 @@
-FROM eclipse-temurin:11-focal
+FROM maven:3-eclipse-temurin-11 AS build
+WORKDIR /usr/src/hobbit-platform
+COPY parent-pom/pom.xml parent-pom/
+RUN mvn --file parent-pom -Dmaven.test.skip=true install
+ARG project=analysis-component
+COPY ${project}/pom.xml ${project}/
+RUN mvn --file ${project} dependency:go-offline
+COPY ${project}/src ${project}/src
+RUN mvn --file ${project} -Dmaven.test.skip=true package
 
-COPY target/analysis-component.jar .
-
-CMD java -cp analysis-component.jar org.hobbit.core.run.ComponentStarter org.hobbit.analysis.AnalysisComponent
+FROM eclipse-temurin:11
+COPY --from=build /usr/src/hobbit-platform/analysis-component/target/analysis-component.jar .
+CMD ["java", "-cp", "analysis-component.jar", "org.hobbit.core.run.ComponentStarter", "org.hobbit.analysis.AnalysisComponent"]

analysis-component/pom.xml

@@ -14,7 +14,7 @@
     <parent>
         <groupId>org.hobbit</groupId>
         <artifactId>parent</artifactId>
-        <version>${hobbitplatform.version}</version>
+        <version>2.0.17</version>
         <relativePath>../parent-pom</relativePath>
     </parent>
     <artifactId>analysis-component</artifactId>

analysis-component/src/main/java/org/hobbit/analysis/AnalysisComponent.java

@@ -16,22 +16,42 @@
  */
 package org.hobbit.analysis;
 
-import java.util.stream.Stream;
-import java.util.stream.Collectors;
 import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import org.aksw.palmetto.evaluate.correlation.PearsonsSampleCorrelationCoefficient;
+import org.apache.commons.io.IOUtils;
 import org.apache.jena.datatypes.RDFDatatype;
 import org.apache.jena.datatypes.xsd.XSDDatatype;
-import org.apache.jena.rdf.model.*;
+import org.apache.jena.rdf.model.Model;
+import org.apache.jena.rdf.model.ModelFactory;
+import org.apache.jena.rdf.model.Property;
+import org.apache.jena.rdf.model.RDFNode;
+import org.apache.jena.rdf.model.ResIterator;
+import org.apache.jena.rdf.model.Resource;
+import org.apache.jena.rdf.model.ResourceFactory;
+import org.apache.jena.rdf.model.Statement;
+import org.apache.jena.rdf.model.StmtIterator;
 import org.apache.jena.vocabulary.DCTerms;
 import org.apache.jena.vocabulary.RDF;
 import org.apache.jena.vocabulary.XSD;
-import org.apache.commons.io.IOUtils;
 import org.hobbit.core.Constants;
 import org.hobbit.core.components.AbstractComponent;
 import org.hobbit.core.data.RabbitQueue;
+import org.hobbit.core.rabbit.QueueingConsumer;
 import org.hobbit.core.rabbit.RabbitMQUtils;
 import org.hobbit.storage.client.StorageServiceClient;
 import org.hobbit.storage.queries.SparqlQueries;
@@ -42,20 +62,31 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.rabbitmq.client.QueueingConsumer;
+import com.rabbitmq.client.Delivery;
 
-import weka.attributeSelection.*;
 import weka.attributeSelection.AttributeSelection;
+import weka.attributeSelection.CfsSubsetEval;
+import weka.attributeSelection.GreedyStepwise;
 import weka.classifiers.functions.LinearRegression;
 import weka.clusterers.SimpleKMeans;
-import weka.core.*;
+import weka.core.Attribute;
+import weka.core.DenseInstance;
+import weka.core.Instance;
+import weka.core.Instances;
+import weka.core.SparseInstance;
 
 
 /**
  * This class implements the functionality for the Analysis Component
  * TODO:: !!REFACTOR INTO A MORE GENERIC DESIGN!!
  */
 public class AnalysisComponent extends AbstractComponent {
+    /**
+     * The time the main receiver thread will sleep if it didn't receive any message
+     * (in seconds).
+     */
+    private static final int WAITING_TIME_BEFORE_CHECKING_STATUS = 60;
+
     private static final Logger LOGGER = LoggerFactory.getLogger(AnalysisComponent.class);
     private static final String GRAPH_URI = Constants.PUBLIC_RESULT_GRAPH_URI;
     protected RabbitQueue controller2AnalysisQueue;
@@ -82,16 +113,24 @@ public void init() throws Exception {
     @Override
     public void run() throws Exception {
         LOGGER.info("Awaiting requests");
-        QueueingConsumer.Delivery delivery;
+        Delivery delivery;
         while (true) {
-            delivery = consumer.nextDelivery();
+            delivery = null;
+            // Let's wait for a delivery for 60 seconds
+            try {
+                delivery = consumer.getDeliveryQueue().poll(WAITING_TIME_BEFORE_CHECKING_STATUS, TimeUnit.SECONDS);
+            } catch (InterruptedException e) {
+                // interrupted; just continue
+            }
             if (delivery != null) {
                 LOGGER.info("Received a request. Processing...");
                 String expUri = RabbitMQUtils.readString(delivery.getBody());
                 handleRequest(expUri);
+            } else {
+                // This would be the place at which we could react to signals, e.g., terminate
+                // the service if needed.
             }
         }
-
     }
 
     protected void handleRequest(String expUri) {
@@ -235,18 +274,17 @@ public void close() throws IOException {
      *            The model of the experiment
      * @return Returns the analyzed model
      */
-
-    /**
+    /*
      private AnalysisModel analyseExperiment(Model experimentModel, String expUri){
      AnalysisModel analysisModel = new AnalysisModel(experimentModel, expUri);
      analysisModel.analyse();
      return analysisModel;
      }*/
 
-    private void notifyQueue(){
+    /*private void notifyQueue(){
         //TODO:: return the status of component
 
-    }
+    }*/
 
     /**
      * This class implements the functionality of the Analysis Model which

docker-compose-dev.yml

@@ -9,16 +9,20 @@ services:
       HOBBIT_RABBIT_IMAGE: "rabbitmq:management"
       HOBBIT_RABBIT_HOST: "rabbit"
       HOBBIT_REDIS_HOST: "redis"
-      DEPLOY_ENV: "testing"
+      DEPLOY_ENV: "develop"
       GITLAB_USER: "${GITLAB_USER}"
       GITLAB_EMAIL: "${GITLAB_EMAIL}"
       GITLAB_TOKEN: "${GITLAB_TOKEN}"
-      LOGGING_GELF_ADDRESS: "udp://localhost:12201"
+      #LOGGING_GELF_ADDRESS: "udp://localhost:12201"
       SWARM_NODE_NUMBER: "1"
       PROMETHEUS_HOST: prometheus
       PROMETHEUS_PORT: 9090
+      USE_GITLAB: "false"
+      LOCAL_METADATA_DIRECTORY: "/metadata"
+      DOCKER_AUTOPULL: "0"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
+      - ./metadata:/metadata
 
   # HOBBIT GUI
   gui:
@@ -36,20 +40,21 @@ services:
       - KEYCLOAK_DIRECT_URL=http://keycloak:8080/auth
       - ELASTICSEARCH_HOST=elasticsearch
       - ELASTICSEARCH_HTTP_PORT=9200
+      - USE_UI_AUTH=false
     #volumes:
       #- /data/docker/messages/global.html:/var/lib/jetty/webapps/messages/global.html
       #- /data/docker/messages/benchmark.html:/var/lib/jetty/webapps/messages/benchmark.html
       #- /data/docker/messages/status.html:/var/lib/jetty/webapps/messages/status.html
 
   # Keycloak user management (used by the GUI)
-  keycloak:
-    image: hobbitproject/hobbit-keycloak:latest
-    ports:
-      - "8181:8080"
-    networks:
-      - hobbit
-    volumes:
-      - ./config/keycloak:/opt/jboss/keycloak/standalone/data/db
+  # keycloak:
+  #   image: hobbitproject/hobbit-keycloak:latest
+  #   ports:
+  #     - "8181:8080"
+  #   networks:
+  #     - hobbit
+  #   volumes:
+  #     - ./config/keycloak:/opt/jboss/keycloak/standalone/data/db
 
   # HOBBIT Analysis component
   analysis:
@@ -90,7 +95,8 @@ services:
     stop_signal: SIGINT
     stop_grace_period: 2m
     volumes:
-      - ./config/db:/opt/virtuoso-opensource/database
+      - ./config/db:/opt/virtuoso-opensource/var/lib/virtuoso/db
+      #- ./config/db:/opt/virtuoso-opensource/database
     networks:
       - hobbit-core
     ports:
@@ -104,16 +110,16 @@ services:
     environment:
       - SPARQL_ENDPOINT_URL=http://vos:8890/sparql
       - HOBBIT_RABBIT_HOST=rabbit
-      - SPARQL_ENDPOINT_USERNAME=HobbitPlatform
-      - SPARQL_ENDPOINT_PASSWORD=Password
+      - SPARQL_ENDPOINT_USERNAME=dba
+      - SPARQL_ENDPOINT_PASSWORD=dba
 
   node-exporter:
     image: prom/node-exporter
     networks:
       - hobbit-core
 
   cadvisor:
-    image: google/cadvisor
+    image: gcr.io/cadvisor/cadvisor
     networks:
       - hobbit-core
     volumes:
@@ -138,8 +144,10 @@ services:
 
 networks:
   hobbit:
-    external:
-      name: hobbit
+    name: hobbit
+    external: true
+    driver: overlay
   hobbit-core:
-    external:
-      name: hobbit-core
+    name: hobbit-core
+    external: true
+    driver: overlay

hobbit-gui/gui-client/src/app/auth/keycloak.service.ts

@@ -18,6 +18,14 @@ export class KeycloakService {
       req.onload = (e) => {
         const keycloakConfig = JSON.parse(req.responseText);
 
+        if (keycloakConfig.clientId === 'DISABLED') {
+          console.log('UI auth is disabled');
+          KeycloakService.auth.loggedIn = true;
+          KeycloakService.auth.authz = true;
+          resolve(null);
+          return;
+        }
+
         const keycloakAuth: any = Keycloak(keycloakConfig);
         keycloakAuth.init({ onLoad: 'login-required' })
           .success(() => {
@@ -66,6 +74,12 @@ export class KeycloakService {
           .error(() => {
             reject('Failed to refresh token');
           });
+      } else {
+        if (KeycloakService.auth.authz === true) {
+          resolve(null);
+          return;
+        }
+        reject('No token');
       }
     });
   }

hobbit-gui/gui-client/src/app/custom-http.service.ts

@@ -93,14 +93,17 @@ export class CustomHttp {
 
     const obs = Observable.fromPromise(this.keycloakService.getToken()).map(token => {
       const requestUrl = environment.backendUrl + url.substr(environment.backendPrefix.length);
-      const headers = { 'Authorization': 'bearer ' + token };
+      const headers = {};
+      if (token !== null) {
+        Object.assign(headers, {Authorization: 'bearer ' + token});
+      }
       let requestOptions = options;
       if (!options)
         requestOptions = { 'headers': new HttpHeaders(headers) };
       else if (!options.headers)
         requestOptions.headers = new HttpHeaders(headers);
       else
-        requestOptions.headers = requestOptions.headers.set('Authorization', headers['Authorization']);
+        Object.assign(requestOptions.headers, headers);
       return { url: requestUrl, options: requestOptions };
     });
     return obs;