Merge pull request #106 from de4a-wp5/iteration2-dev

Iteration2 dev

Jenkinsfile

@@ -62,6 +62,7 @@ pipeline {
                     }
                 }
                 sh 'docker system prune -f'
+				sh 'docker network create docker-ci_default'
             }
         }
 

README.md

@@ -31,7 +31,12 @@ Checkout the technical documentation on [the Wiki page](https://wiki.de4a.eu/ind
 #### Service APIs
 
 * **`/service/ial/{cot}`** - Query the IAL for provided Canonical Object Types (COTs). Multiple COTs can be separated by `,`. No ATU restrictions are applied.
+    * Optional query parameter **`environment`** - The runtime environment for which the Connector will filter the result entries from IAL. The following environments are allowed.
+        * The `playground` environment only allows `9999:.+mock-it2` identifier values
+        * The `test` environment only allows `99\d\d:.+test-it2` identifier values
+        * The `pilot` environment allows for all other identifiers (so the ones that are neither `playground` nor `test`)
 * **`/service/ial/{cot}/{atu}`** - Query the IAL for provided Canonical Object Types (COTs) but only for the ones in the specified ATU. Multiple COTs can be separated by `,`.
+    * Optional query parameter **`environment`** - The runtime environment for which the Connector will filter the result entries from IAL. See above for details.
 * **`/service/reload-addresses`** - Reload the internal address list for forwarding to DE and DO from the backend file.
 
 ### Configuration
@@ -263,6 +268,8 @@ Once you have deployed the `war` file, there are several **checks to ensure that
 
 # News and Noteworthy
 
+* v0.2.5 - 2022-08-24
+    * Extended the `/service/ial` queries with an optional `environment` request parameter that can limit the result DOs
 * v0.2.4 - 2022-08-19
     * Fixed a Spring problem with the `/service/ial` APIs when run in a Docker container (same name of resolved method)
 * v0.2.3 - 2022-08-18

build-and-push-docker-release.cmd

@@ -1,7 +1,7 @@
 @echo off
 
 :: The version to release
-set VER=0.2.4
+set VER=0.2.5
 
 cd de4a-connector
 docker build --pull -t de4a/connector:%VER% .

compile-build-and-run-docker-snapshot.cmd

@@ -17,4 +17,4 @@ goto end
 :error
 pause
 
-:end
+:end

de4a-connector/Dockerfile

@@ -1,17 +1,25 @@
-FROM tomcat:9-jdk11
+# Part 1:
+
+FROM ubuntu:latest as build
 
 # Install unzip
 RUN apt-get update \
   && apt-get upgrade -y \
   && apt-get install -y unzip \
   && rm -rf /var/lib/apt/lists/*
 
+COPY target/de4a-connector.war ./connector.war
+
+RUN unzip connector.war -d /smp && \
+    rm /smp/WEB-INF/classes/*.p12
+
+# Part 2:
+
+FROM tomcat:9-jdk11
+
 ENV CATALINA_OPTS="$CATALINA_OPTS -Djava.security.egd=file:/dev/urandom"
 
 WORKDIR $CATALINA_HOME/webapps
+RUN rm -rf manager host-manager docs examples ROOT
 
-COPY target/de4a-connector.war ./connector.war
-
-RUN rm -rf manager host-manager docs examples ROOT && \
-    unzip connector.war -d ROOT  && \
-    rm -f connector.war
+COPY --from=build /smp $CATALINA_HOME/webapps/ROOT

de4a-connector/src/main/java/eu/de4a/connector/EDE4ARuntimeEnvironment.java

@@ -0,0 +1,60 @@
+package eu.de4a.connector;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import com.helger.commons.annotation.Nonempty;
+import com.helger.commons.id.IHasID;
+import com.helger.commons.lang.EnumHelper;
+import com.helger.commons.regex.RegExHelper;
+import com.helger.peppolid.IParticipantIdentifier;
+
+public enum EDE4ARuntimeEnvironment implements IHasID <String>
+{
+  PLAYGROUND ("playground") {
+    @Override
+    public boolean isAllowedParticipantID (@Nonnull final IParticipantIdentifier aPI)
+    {
+      final String sValue = aPI.getValue ();
+      return RegExHelper.stringMatchesPattern ("9999:.+-mock-it2", sValue);
+    }
+  },
+  TEST ("test") {
+    @Override
+    public boolean isAllowedParticipantID (@Nonnull final IParticipantIdentifier aPI)
+    {
+      final String sValue = aPI.getValue ();
+      return RegExHelper.stringMatchesPattern ("99\\d\\d:.+-test-it2", sValue);
+    }
+  },
+  PILOT ("pilot") {
+    @Override
+    public boolean isAllowedParticipantID (@Nonnull final IParticipantIdentifier aPI)
+    {
+      // The ones that are neither playground nor test are pilot ones :)
+      return !PLAYGROUND.isAllowedParticipantID (aPI) && !TEST.isAllowedParticipantID (aPI);
+    }
+  };
+
+  private final String m_sID;
+
+  EDE4ARuntimeEnvironment (@Nonnull @Nonempty final String sID)
+  {
+    m_sID = sID;
+  }
+
+  @Nonnull
+  @Nonempty
+  public String getID ()
+  {
+    return m_sID;
+  }
+
+  public abstract boolean isAllowedParticipantID (@Nonnull IParticipantIdentifier aPI);
+
+  @Nullable
+  public static EDE4ARuntimeEnvironment getFromIDOrNull (@Nullable final String sID)
+  {
+    return EnumHelper.getFromIDOrNull (EDE4ARuntimeEnvironment.class, sID);
+  }
+}

de4a-connector/src/main/java/eu/de4a/connector/api/controller/ResponseController.java

@@ -133,9 +133,10 @@ public class ResponseController
                                                         docTypeID,
                                                         DE4AConstants.PROCESS_ID_RESPONSE);
 
-    //TODO Get Subscription response and extract evidences/errors, actually the ResponseEventSubscriptionItemType does not have ErrorType
+    responseObj.getResponseEventSubscriptionItemAtIndex(0).getError();
+    String responseMetadata = MessageUtils.getEventSubscriptionResponseMetadata(responseObj.getResponseEventSubscriptionItem());
     this.apiManager.processIncomingMessage (ELogMessage.LOG_RES_SUBSC_DO_DT, 
-    		responseObj, messageDTO, marshaller, responseObj.getRequestId());
+    		responseObj, messageDTO, marshaller, responseObj.getRequestId(), responseMetadata);
 
     return ResponseEntity.status (HttpStatus.OK).body (ConnectorExceptionHandler.getSuccessResponseBytes ());
   }

de4a-connector/src/main/java/eu/de4a/connector/api/controller/ServiceController.java

@@ -1,5 +1,7 @@
 package eu.de4a.connector.api.controller;
 
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import javax.validation.Valid;
 import javax.validation.constraints.NotNull;
 
@@ -14,17 +16,25 @@
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
 
+import com.helger.commons.collection.impl.CommonsArrayList;
 import com.helger.commons.string.StringHelper;
+import com.helger.dcng.api.DcngConfig;
 import com.helger.dcng.core.api.DcngApiHelper;
+import com.helger.peppolid.IParticipantIdentifier;
 
+import eu.de4a.connector.EDE4ARuntimeEnvironment;
 import eu.de4a.connector.error.exceptions.ConnectorException;
 import eu.de4a.connector.error.model.EFamilyErrorType;
 import eu.de4a.connector.error.model.ELayerError;
 import eu.de4a.connector.utils.KafkaClientWrapper;
 import eu.de4a.connector.utils.ServiceUtils;
 import eu.de4a.ial.api.IALMarshaller;
+import eu.de4a.ial.api.jaxb.ErrorType;
+import eu.de4a.ial.api.jaxb.ResponseItemType;
 import eu.de4a.ial.api.jaxb.ResponseLookupRoutingInformationType;
+import eu.de4a.ial.api.jaxb.ResponsePerCountryType;
 import eu.de4a.kafkaclient.model.EExternalModule;
 
 @Controller
@@ -37,54 +47,144 @@ public class ServiceController
   @Autowired
   private ServiceUtils serviceUtils;
 
+  @Nonnull
+  private static ErrorType _createError (@Nonnull final String sCode, @Nonnull final String sMsg)
+  {
+    final ErrorType ret = new ErrorType ();
+    ret.setCode (sCode);
+    ret.setText (sMsg);
+    return ret;
+  }
+
+  private static void _filterDOs (@Nonnull final ResponseLookupRoutingInformationType aResponse,
+                                  @Nullable final String environment,
+                                  @Nullable final String sCOTIDs,
+                                  @Nullable final String sATUCode)
+  {
+    // Filter only successful responses
+    if (aResponse.hasResponseItemEntries ())
+    {
+      // Check if the environment is OKAY
+      final EDE4ARuntimeEnvironment rtEnv = EDE4ARuntimeEnvironment.getFromIDOrNull (environment);
+      if (rtEnv != null)
+      {
+        if (LOGGER.isInfoEnabled ())
+          LOGGER.info ("Filtering allowed DOs returned by /service/ial/* query to " + rtEnv);
+
+        for (final ResponseItemType aRI : new CommonsArrayList <> (aResponse.getResponseItem ()))
+        {
+          for (final ResponsePerCountryType aRPC : new CommonsArrayList <> (aRI.getResponsePerCountry ()))
+          {
+            aRPC.getProvision ().removeIf (aProv -> {
+              final IParticipantIdentifier aPI = DcngConfig.getIdentifierFactory ()
+                                                           .parseParticipantIdentifier (aProv.getDataOwnerId ());
+              if (rtEnv.isAllowedParticipantID (aPI))
+                return false;
+              if (LOGGER.isInfoEnabled ())
+                LOGGER.info ("Ignoring non-" + rtEnv + " DataOwner ID " + aProv.getDataOwnerId ());
+              return true;
+            });
+            if (aRPC.hasNoProvisionEntries ())
+              aRI.getResponsePerCountry ().remove (aRPC);
+          }
+          if (aRI.hasNoResponsePerCountryEntries ())
+            aResponse.getResponseItem ().remove (aRI);
+        }
+
+        // We now might have an empty response
+        if (aResponse.hasNoResponseItemEntries ())
+        {
+          aResponse.addError (_createError ("no-match",
+                                            "Found NO matches searching for '" +
+                                                        sCOTIDs +
+                                                        "'" +
+                                                        (sATUCode != null ? " and ATU code '" + sATUCode + "'" : "") +
+                                                        " after filtering for " +
+                                                        rtEnv));
+        }
+      }
+      else
+      {
+        if (StringHelper.hasText (environment))
+          if (LOGGER.isWarnEnabled ())
+            LOGGER.warn ("Unsupported environment '" + environment + "' provided to /service/ial/* query");
+      }
+    }
+  }
+
   @GetMapping (value = "/ial/{cot}", produces = MediaType.APPLICATION_XML_VALUE)
-  public ResponseEntity <byte []> callIalCot (@Valid @PathVariable @NotNull final String cot)
+  public ResponseEntity <byte []> callIalCot (@Valid @PathVariable @NotNull final String cot,
+                                              @RequestParam (required = false) final String environment)
   {
     if (LOGGER.isInfoEnabled ())
-      LOGGER.info ("Request to API '/service/ial/" + cot + "' received");
+      LOGGER.info ("Request to API '/service/ial/" +
+                   cot +
+                   "' " +
+                   (StringHelper.hasText (environment) ? "for environment '" + environment + "' " : "") +
+                   "received");
 
     // Main query
-    final ResponseLookupRoutingInformationType aResponse = DcngApiHelper.queryIAL (StringHelper.getExplodedToOrderedSet (",", cot));
+    final ResponseLookupRoutingInformationType aResponse = DcngApiHelper.queryIAL (StringHelper.getExplodedToOrderedSet (",",
+                                                                                                                         cot));
     if (aResponse == null)
     {
-    	final String errorMsg = "Error querying IAL without ATU code";
-    	KafkaClientWrapper.sendError(EFamilyErrorType.CONNECTION_ERROR, EExternalModule.CONNECTOR_DR, EExternalModule.IAL.getLabel(), errorMsg);
-
-    	// Error case
-    	throw new ConnectorException ().withFamily (EFamilyErrorType.CONNECTION_ERROR)
+      final String errorMsg = "Error querying IAL without ATU code";
+      KafkaClientWrapper.sendError (EFamilyErrorType.CONNECTION_ERROR,
+                                    EExternalModule.CONNECTOR_DR,
+                                    EExternalModule.IAL.getLabel (),
+                                    errorMsg);
+
+      // Error case
+      throw new ConnectorException ().withFamily (EFamilyErrorType.CONNECTION_ERROR)
                                      .withLayer (ELayerError.INTERNAL_FAILURE)
                                      .withModule (EExternalModule.IAL)
                                      .withMessageArg (errorMsg)
                                      .withHttpStatus (HttpStatus.INTERNAL_SERVER_ERROR);
     }
 
+    _filterDOs (aResponse, environment, cot, null);
+
     // Success case
     return ResponseEntity.status (HttpStatus.OK)
                          .body (IALMarshaller.responseLookupRoutingInformationMarshaller ().getAsBytes (aResponse));
   }
 
   @GetMapping (value = "/ial/{cot}/{atu}", produces = MediaType.APPLICATION_XML_VALUE)
   public ResponseEntity <byte []> callIalCotAtu (@Valid @PathVariable @NotNull final String cot,
-                                                 @Valid @PathVariable @NotNull final String atu)
+                                                 @Valid @PathVariable @NotNull final String atu,
+                                                 @RequestParam (required = false) final String environment)
   {
     if (LOGGER.isInfoEnabled ())
-      LOGGER.info ("Request to API '/service/ial/" + cot + "/" + atu + "' received");
+      LOGGER.info ("Request to API '/service/ial/" +
+                   cot +
+                   "/" +
+                   atu +
+                   "' " +
+                   (StringHelper.hasText (environment) ? "for environment '" + environment + "' " : "") +
+                   "received");
 
     // Main query
-    final ResponseLookupRoutingInformationType aResponse = DcngApiHelper.queryIAL (StringHelper.getExplodedToOrderedSet (",", cot), atu);
+    final ResponseLookupRoutingInformationType aResponse = DcngApiHelper.queryIAL (StringHelper.getExplodedToOrderedSet (",",
+                                                                                                                         cot),
+                                                                                   atu);
     if (aResponse == null)
     {
-    	final String errorMsg = "Error querying IAL with ATU code";
-    	KafkaClientWrapper.sendError(EFamilyErrorType.CONNECTION_ERROR, EExternalModule.CONNECTOR_DR, EExternalModule.IAL.getLabel(), errorMsg);
-
-    	// Error case
-    	throw new ConnectorException ().withFamily (EFamilyErrorType.CONNECTION_ERROR)
+      final String errorMsg = "Error querying IAL with ATU code";
+      KafkaClientWrapper.sendError (EFamilyErrorType.CONNECTION_ERROR,
+                                    EExternalModule.CONNECTOR_DR,
+                                    EExternalModule.IAL.getLabel (),
+                                    errorMsg);
+
+      // Error case
+      throw new ConnectorException ().withFamily (EFamilyErrorType.CONNECTION_ERROR)
                                      .withLayer (ELayerError.INTERNAL_FAILURE)
                                      .withModule (EExternalModule.IAL)
                                      .withMessageArg (errorMsg)
                                      .withHttpStatus (HttpStatus.INTERNAL_SERVER_ERROR);
     }
 
+    _filterDOs (aResponse, environment, cot, atu);
+
     // Success case
     return ResponseEntity.status (HttpStatus.OK)
                          .body (IALMarshaller.responseLookupRoutingInformationMarshaller ().getAsBytes (aResponse));

de4a-connector/src/main/java/eu/de4a/connector/api/manager/MessageExchangeManager.java

@@ -233,12 +233,8 @@ public void processMessageExchange (@Nonnull final MessageExchangeWrapper messag
 
               case "ResponseEventSubscription":
                 logMessage = ELogMessage.LOG_RES_SUBSC_DT_DR;
-                // var res =
-                // DE4ACoreMarshaller.dtResponseEventSubscriptionMarshaller().read
-                // (aRegRepElement);
-                // metadata =
-                // MessageUtils.getEventSubscriptionResponseMetadata(res.getResponseEventSubscriptionItem());
-                // //TODO method to be implemented
+                var res = DE4ACoreMarshaller.dtResponseEventSubscriptionMarshaller().read(aRegRepElement);
+                metadata = MessageUtils.getEventSubscriptionResponseMetadata(res.getResponseEventSubscriptionItem());
                 break;
 
               case "EventNotification":
@@ -253,16 +249,6 @@ public void processMessageExchange (@Nonnull final MessageExchangeWrapper messag
           }
         }
 
-        //final String sRequestID = DOMUtils.getValueFromXpath (XPATH_REQUEST_ID, aTargetDoc.getDocumentElement ());
-        /*
-        KafkaClientWrapper.sendInfo (logMessage,
-        							 eMessageServiceType.getType (),
-                                     sRequestID,
-                                     senderID,
-                                     receiverID,
-                                     metadata);
-                                     */
-
         response = this.deliverService.pushMessage (eMessageServiceType,
                                                     aTargetDoc,
                                                     senderID,