Skip to content

Commit

Permalink
Merge pull request #2025 from eclipse/GH-1900-output-shacl-validation…
Browse files Browse the repository at this point in the history 
…-report-from-server

GH-1900 Server responds with SHACL validation report if validation failed
  • Loading branch information
@hmottestad
hmottestad authored Mar 31, 2020
2 parents 0d61892 + 9681889 commit 4e10890
Show file tree
Hide file tree
Showing 27 changed files with 834 additions and 144 deletions.
18 changes: 18 additions & 0 deletions core/http/client/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,24 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>rdf4j-rio-nquads</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>rdf4j-rio-jsonld</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>rdf4j-rio-turtle</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
Expand Down
144 changes: 106 additions & 38 deletions core/http/client/src/main/java/org/eclipse/rdf4j/http/client/SPARQLProtocolSession.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,23 +7,6 @@
*******************************************************************************/
package org.eclipse.rdf4j.http.client;

import static org.eclipse.rdf4j.http.protocol.Protocol.ACCEPT_PARAM_NAME;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URISyntaxException;
import java.nio.charset.Charset;
import java.nio.charset.IllegalCharsetNameException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;

import org.apache.commons.io.IOUtils;
import org.apache.http.Header;
import org.apache.http.HeaderElement;
Expand Down Expand Up @@ -55,6 +38,7 @@
import org.apache.http.util.EntityUtils;
import org.eclipse.rdf4j.RDF4JConfigException;
import org.eclipse.rdf4j.RDF4JException;
import org.eclipse.rdf4j.http.client.shacl.RemoteShaclValidationException;
import org.eclipse.rdf4j.http.protocol.Protocol;
import org.eclipse.rdf4j.http.protocol.UnauthorizedException;
import org.eclipse.rdf4j.http.protocol.error.ErrorInfo;
Expand Down Expand Up @@ -102,6 +86,26 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.StringReader;
import java.net.HttpURLConnection;
import java.net.URISyntaxException;
import java.nio.charset.Charset;
import java.nio.charset.IllegalCharsetNameException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutorService;

import static org.eclipse.rdf4j.http.protocol.Protocol.ACCEPT_PARAM_NAME;

/**
* The SPARQLProtocolSession provides low level HTTP methods for communication with SPARQL endpoints. All methods are
* compliant to the <a href="https://www.w3.org/TR/sparql11-protocol/">SPARQL 1.1 Protocol W3C Recommendation</a>.
Expand All @@ -113,7 +117,7 @@
* <p/>
* Functionality specific to the RDF4J HTTP protocol can be found in {@link RDF4JProtocolSession} (which is used by
* HTTPRepository).
*
*
* @author Herko ter Horst
* @author Arjohn Kampman
* @author Andreas Schwarte
Expand Down Expand Up @@ -153,7 +157,7 @@ public class SPARQLProtocolSession implements HttpClientDependent, AutoCloseable
*/
private final int maximumUrlLength;

final Logger logger = LoggerFactory.getLogger(SPARQLProtocolSession.class);
final static Logger logger = LoggerFactory.getLogger(SPARQLProtocolSession.class);

/*-----------*
* Variables *
Expand Down Expand Up @@ -250,7 +254,7 @@ protected void setUpdateURL(String updateURL) {
/**
* Sets the preferred format for encoding tuple query results. The {@link TupleQueryResultFormat#BINARY binary}
* format is preferred by default.
*
*
* @param format The preferred {@link TupleQueryResultFormat}, or <tt>null</tt> to indicate no specific format is
* preferred.
*/
Expand All @@ -260,7 +264,7 @@ public void setPreferredTupleQueryResultFormat(TupleQueryResultFormat format) {

/**
* Gets the preferred {@link TupleQueryResultFormat} for encoding tuple query results.
*
*
* @return The preferred format, of <tt>null</tt> if no specific format is preferred.
*/
public TupleQueryResultFormat getPreferredTupleQueryResultFormat() {
Expand All @@ -270,7 +274,7 @@ public TupleQueryResultFormat getPreferredTupleQueryResultFormat() {
/**
* Sets the preferred format for encoding RDF documents. The {@link RDFFormat#TURTLE Turtle} format is preferred by
* default.
*
*
* @param format The preferred {@link RDFFormat}, or <tt>null</tt> to indicate no specific format is preferred.
*/
public void setPreferredRDFFormat(RDFFormat format) {
Expand All @@ -279,7 +283,7 @@ public void setPreferredRDFFormat(RDFFormat format) {

/**
* Gets the preferred {@link RDFFormat} for encoding RDF documents.
*
*
* @return The preferred format, of <tt>null</tt> if no specific format is preferred.
*/
public RDFFormat getPreferredRDFFormat() {
Expand All @@ -289,7 +293,7 @@ public RDFFormat getPreferredRDFFormat() {
/**
* Sets the preferred format for encoding boolean query results. The {@link BooleanQueryResultFormat#TEXT binary}
* format is preferred by default.
*
*
* @param format The preferred {@link BooleanQueryResultFormat}, or <tt>null</tt> to indicate no specific format is
* preferred.
*/
Expand All @@ -299,7 +303,7 @@ public void setPreferredBooleanQueryResultFormat(BooleanQueryResultFormat format

/**
* Gets the preferred {@link BooleanQueryResultFormat} for encoding boolean query results.
*
*
* @return The preferred format, of <tt>null</tt> if no specific format is preferred.
*/
public BooleanQueryResultFormat getPreferredBooleanQueryResultFormat() {
Expand All @@ -308,7 +312,7 @@ public BooleanQueryResultFormat getPreferredBooleanQueryResultFormat() {

/**
* Set the username and password for authentication with the remote server.
*
*
* @param username the username
* @param password the password
*/
Expand Down Expand Up @@ -450,7 +454,7 @@ public boolean sendBooleanQuery(QueryLanguage ql, String query, String baseURI,

/**
* Get the additional HTTP headers which will be used
*
*
* @return a read-only view of the additional HTTP headers which will be included in every request to the server.
*/
public Map<String, String> getAdditionalHttpHeaders() {
Expand All @@ -460,7 +464,7 @@ public Map<String, String> getAdditionalHttpHeaders() {
/**
* Set additional HTTP headers to be included in every request to the server, which may be required for certain
* unusual server configurations.
*
*
* @param additionalHttpHeaders a map containing pairs of header names and values. May be null
*/
public void setAdditionalHttpHeaders(Map<String, String> additionalHttpHeaders) {
Expand All @@ -479,8 +483,9 @@ protected HttpUriRequest getQueryMethod(QueryLanguage ql, String query, String b
String queryUrlWithParams;
try {
URIBuilder urib = new URIBuilder(getQueryURL());
for (NameValuePair nvp : queryParams)
for (NameValuePair nvp : queryParams) {
urib.addParameter(nvp.getName(), nvp.getValue());
}
queryUrlWithParams = urib.toString();
} catch (URISyntaxException e) {
throw new AssertionError(e);
Expand All @@ -506,7 +511,7 @@ protected HttpUriRequest getQueryMethod(QueryLanguage ql, String query, String b

/**
* Return whether the provided query should use POST (otherwise use GET)
*
*
* @param fullQueryUrl the complete URL, including hostname and all HTTP query parameters
*/
protected boolean shouldUsePost(String fullQueryUrl) {
Expand All @@ -530,8 +535,9 @@ protected HttpUriRequest getUpdateMethod(QueryLanguage ql, String update, String
method.setEntity(new UrlEncodedFormEntity(queryParams, UTF8));

if (this.additionalHttpHeaders != null) {
for (Map.Entry<String, String> additionalHeader : additionalHttpHeaders.entrySet())
for (Map.Entry<String, String> additionalHeader : additionalHttpHeaders.entrySet()) {
method.addHeader(additionalHeader.getKey(), additionalHeader.getValue());
}
}

return method;
Expand Down Expand Up @@ -729,7 +735,7 @@ protected void getTupleQueryResult(HttpUriRequest method, TupleQueryResultHandle
* Send the tuple query via HTTP and throws an exception in case anything goes wrong, i.e. only for HTTP 200 the
* method returns without exception. If HTTP status code is not equal to 200, the request is aborted, however pooled
* connections are not released.
*
*
* @param method
* @throws RepositoryException
* @throws HttpException
Expand Down Expand Up @@ -908,7 +914,7 @@ private HttpResponse sendGraphQueryViaHttp(HttpUriRequest method, boolean requir
/**
* Parse the response in this thread using a suitable {@link BooleanQueryResultParser}. All HTTP connections are
* closed and released in this method
*
*
* @throws RDF4JException
*/
protected boolean getBoolean(HttpUriRequest method) throws IOException, RDF4JException {
Expand Down Expand Up @@ -975,7 +981,7 @@ private HttpResponse sendBooleanQueryViaHttp(HttpUriRequest method, Set<QueryRes
/**
* Convenience method to deal with HTTP level errors of tuple, graph and boolean queries in the same way. This
* method aborts the HTTP connection.
*
*
* @param method
* @throws RDF4JException
*/
Expand Down Expand Up @@ -1041,6 +1047,11 @@ protected HttpResponse execute(HttpUriRequest method) throws IOException, RDF4JE
throw new MalformedQueryException(errInfo.getErrorMessage());
} else if (errInfo.getErrorType() == ErrorType.UNSUPPORTED_QUERY_LANGUAGE) {
throw new UnsupportedQueryLanguageException(errInfo.getErrorMessage());
} else if (contentTypeIs(response, "application/shacl-validation-report")) {
RDFFormat format = getContentTypeSerialisation(response);
throw new RepositoryException(new RemoteShaclValidationException(
new StringReader(errInfo.toString()), "", format));

} else if (errInfo.toString().length() > 0) {
throw new RepositoryException(errInfo.toString());
} else {
Expand All @@ -1055,6 +1066,63 @@ protected HttpResponse execute(HttpUriRequest method) throws IOException, RDF4JE
}
}

static RDFFormat getContentTypeSerialisation(HttpResponse response) {
Header[] headers = response.getHeaders("Content-Type");

Set<RDFFormat> rdfFormats = RDFParserRegistry.getInstance().getKeys();
if (rdfFormats.isEmpty()) {
throw new RepositoryException("No tuple RDF parsers have been registered");
}

for (Header header : headers) {
for (HeaderElement element : header.getElements()) {
// SHACL Validation report Content-Type gets transformed from:
// application/shacl-validation-report+n-quads => application/n-quads
// application/shacl-validation-report+ld+json => application/ld+json
// text/shacl-validation-report+turtle => text/turtle

String[] split = element.getName().split("\\+");
StringBuilder serialisation = new StringBuilder(element.getName().split("/")[0] + "/");
for (int i = 1; i < split.length; i++) {
serialisation.append(split[i]);
if (i + 1 < split.length) {
serialisation.append("+");
}
}

logger.debug("SHACL validation report is serialised as: " + serialisation.toString());

Optional<RDFFormat> rdfFormat = RDFFormat.matchMIMEType(serialisation.toString(), rdfFormats);

if (rdfFormat.isPresent()) {
return rdfFormat.get();
}
}
}

throw new RepositoryException("Unsupported content-type for SHACL Validation Report: "
+ Arrays.toString(response.getHeaders("Content-Type")));

}

private static boolean contentTypeIs(HttpResponse response, String contentType) {
Header[] headers = response.getHeaders("Content-Type");
if (headers.length == 0) {
return false;
}

for (Header header : headers) {
for (HeaderElement element : header.getElements()) {
String name = element.getName().split("\\+")[0];
if (contentType.equals(name)) {
return true;
}
}
}

return false;
}

/*-------------------------*
* General utility methods *
*-------------------------*/
Expand All @@ -1063,7 +1131,7 @@ protected HttpResponse execute(HttpUriRequest method) throws IOException, RDF4JE
* Gets the MIME type specified in the response headers of the supplied method, if any. For example, if the response
* headers contain <tt>Content-Type: application/xml;charset=UTF-8</tt>, this method will return
* <tt>application/xml</tt> as the MIME type.
*
*
* @param method The method to get the reponse MIME type from.
* @return The response MIME type, or <tt>null</tt> if not available.
*/
Expand All @@ -1076,7 +1144,7 @@ protected String getResponseMIMEType(HttpResponse method) throws IOException {
for (HeaderElement headerEl : headerElements) {
String mimeType = headerEl.getName();
if (mimeType != null) {
logger.debug("reponse MIME type is {}", mimeType);
logger.debug("response MIME type is {}", mimeType);
return mimeType;
}
}
Expand All @@ -1098,7 +1166,7 @@ protected ErrorInfo getErrorInfo(HttpResponse response) throws RepositoryExcepti

/**
* Sets the parser configuration used to process HTTP response data.
*
*
* @param parserConfig The parserConfig to set.
*/
public void setParserConfig(ParserConfig parserConfig) {
Expand All @@ -1124,7 +1192,7 @@ public long getConnectionTimeout() {

/**
* Sets the http connection read timeout.
*
*
* @param timeout timeout in milliseconds. Zero sets to infinity.
*/
public void setConnectionTimeout(long timeout) {
Expand Down
43 changes: 43 additions & 0 deletions ...ent/src/main/java/org/eclipse/rdf4j/http/client/shacl/RemoteShaclValidationException.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
/*******************************************************************************
* Copyright (c) 2020 Eclipse RDF4J contributors.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Distribution License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/org/documents/edl-v10.php.
*******************************************************************************/

package org.eclipse.rdf4j.http.client.shacl;

import org.eclipse.rdf4j.RDF4JException;
import org.eclipse.rdf4j.common.annotation.Experimental;
import org.eclipse.rdf4j.exceptions.ValidationException;
import org.eclipse.rdf4j.model.Model;
import org.eclipse.rdf4j.model.vocabulary.SHACL;
import org.eclipse.rdf4j.rio.RDFFormat;

import java.io.StringReader;

/**
* Experimental support for handling SHACL violations against a remote RDF4J server.
*/
@Experimental
public class RemoteShaclValidationException extends RDF4JException implements ValidationException {

private final RemoteValidation remoteValidation;

public RemoteShaclValidationException(StringReader stringReader, String s, RDFFormat format) {
remoteValidation = new RemoteValidation(stringReader, s, format);
}

/**
* @return A Model containing the validation report as specified by the SHACL Recommendation
*/
@Override
public Model validationReportAsModel() {
Model model = remoteValidation.asModel();
model.setNamespace(SHACL.PREFIX, SHACL.NAMESPACE);
return model;

}

}
Loading

0 comments on commit 4e10890

Please sign in to comment.