From 9fe0fe3970a4cdacae2468b75dc5138b8587e08a Mon Sep 17 00:00:00 2001 From: Martijn Maas Date: Fri, 30 Nov 2018 13:43:03 +0100 Subject: [PATCH 1/4] Add poolparty plugin --- conf/termennetwerk.xml | 6 ++ pom.xml | 11 +++ .../nl/knaw/huc/di/nde/recipe/PoolParty.java | 89 +++++++++++++++++++ 3 files changed, 106 insertions(+) create mode 100644 src/main/java/nl/knaw/huc/di/nde/recipe/PoolParty.java diff --git a/conf/termennetwerk.xml b/conf/termennetwerk.xml index a3e5855..116659b 100644 --- a/conf/termennetwerk.xml +++ b/conf/termennetwerk.xml @@ -10,4 +10,10 @@ http://openskos.beeldengeluid.nl/api http://data.beeldengeluid.nl/gtaa/Onderwerpen + + https://data.cultureelerfgoed.nl/PoolParty/sparql/term/id/cht + + PREFIX%20skos%3A%3Chttp%3A%2F%2Fwww.w3.org%2F2004%2F02%2Fskos%2Fcore%23%3E%0A%0ASELECT%20%3Furi%20%3FprefLabel%20%3FaltLabel%20%3FhiddenLabel%20%3FscopeNote%0AWHERE%20%7B%0A%20%20%3Furi%20%3Fx%20skos%3AConcept%20.%0A%20%20%3Furi%20skos%3AprefLabel%20%3FprefLabel%20.%0A%20%20OPTIONAL%20%7B%0A%20%20%20%20%3Fsubject%20skos%3AaltLabel%20%3FaltLabel%20.%0A%20%20%20%20%3Fsubject%20skos%3AhiddenLabel%20%3FhiddenLabel%20.%0A%20%20%20%20%3Fsubject%20skos%3AscopeNote%20%3FscopeNote%20.%0A%20%20%7D%0A%20%20FILTER%20%28LANG%28%3FprefLabel%29%20%3D%20%22nl%22%29%20.%0A%20%20FILTER%20%28CONTAINS%28LCASE%28%3FprefLabel%29%2C%20LCASE%28%22${match}%22%29%29%29%0A%7D%0AORDER%20BY%20%3FprefLabel%0ALIMIT%2010%0A + + diff --git a/pom.xml b/pom.xml index 7d7ed4b..165df77 100644 --- a/pom.xml +++ b/pom.xml @@ -76,6 +76,17 @@ 1.0-SNAPSHOT jar + + org.apache.httpcomponents + httpclient + 4.5.6 + + + org.apache.httpcomponents + httpmime + 4.5.6 + + diff --git a/src/main/java/nl/knaw/huc/di/nde/recipe/PoolParty.java b/src/main/java/nl/knaw/huc/di/nde/recipe/PoolParty.java new file mode 100644 index 0000000..8b6add8 --- /dev/null +++ b/src/main/java/nl/knaw/huc/di/nde/recipe/PoolParty.java @@ -0,0 +1,89 @@ +package nl.knaw.huc.di.nde.recipe; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.google.common.collect.Lists; +import net.sf.saxon.s9api.SaxonApiException; +import net.sf.saxon.s9api.XdmItem; +import nl.knaw.huc.di.nde.TermDTO; +import nl.mpi.tla.util.Saxon; +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.StringWriter; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.List; + +public class PoolParty implements RecipeInterface { + + public static final Logger LOG = LoggerFactory.getLogger(PoolParty.class); + public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + @Override + public List fetchMatchingTerms(XdmItem config, String match) { + + ArrayList terms = Lists.newArrayList(); + try { + StringBuilder api = new StringBuilder(Saxon.xpath2string(config, "nde:api", null, OpenSKOS.NAMESPACES)); + String query = Saxon.xpath2string(config, "nde:query", null, OpenSKOS.NAMESPACES); + + URLEncoder.encode(query, "UTF-8"); + api.append("?format=application/json&query=").append(query.replace("${match}", match).trim()); + + CloseableHttpClient client = HttpClients.createDefault(); + HttpGet httpGet = new HttpGet(api.toString()); + CloseableHttpResponse httpResponse = client.execute(httpGet); + String theString = entityToString(httpResponse.getEntity()); + // LOG.info("response {}", theString); + JsonNode jsonNode = OBJECT_MAPPER.readTree(theString); + LOG.info("json results: {}", jsonNode.get("results")); + LOG.info("json bindings: {}", jsonNode.get("results").get("bindings")); + ArrayNode results = (ArrayNode) jsonNode.get("results").get("bindings"); + results.iterator().forEachRemaining(res -> { + TermDTO term = new TermDTO(); + String uri = res.get("uri").get("value").asText(); + try { + term.uri = new URI(uri); + term.definition = getOptional(res, "hiddenLabel"); + term.altLabel = getOptional(res, "altLabel"); + term.prefLabel = Lists.newArrayList(res.get("prefLabel").get("value").textValue()); + term.scopeNote = getOptional(res, "scopeNote"); + } catch (URISyntaxException e) { + LOG.error("Uri {} not an URI", uri); + } + + terms.add(term); + }); + + } catch (IOException | SaxonApiException e) { + LOG.error("Request failed: ", e); + } + + + return terms; + } + + private ArrayList getOptional(JsonNode res, String field) { + if (res.has(field)) { + return Lists.newArrayList(res.get(field).get("value").asText()); + } + return Lists.newArrayList(); + } + + private String entityToString(HttpEntity responseEntity) throws IOException { + StringWriter writer = new StringWriter(); + IOUtils.copy(responseEntity.getContent(), writer, "UTF-8"); + return writer.toString(); + } +} From 880bd888c8dbd632c1c43ffce791bd5fadadfe9c Mon Sep 17 00:00:00 2001 From: Martijn Maas Date: Fri, 30 Nov 2018 13:56:34 +0100 Subject: [PATCH 2/4] Make POST request --- .../nl/knaw/huc/di/nde/recipe/PoolParty.java | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/main/java/nl/knaw/huc/di/nde/recipe/PoolParty.java b/src/main/java/nl/knaw/huc/di/nde/recipe/PoolParty.java index 8b6add8..dafabf5 100644 --- a/src/main/java/nl/knaw/huc/di/nde/recipe/PoolParty.java +++ b/src/main/java/nl/knaw/huc/di/nde/recipe/PoolParty.java @@ -10,10 +10,12 @@ import nl.mpi.tla.util.Saxon; import org.apache.commons.io.IOUtils; import org.apache.http.HttpEntity; +import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicNameValuePair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,34 +23,38 @@ import java.io.StringWriter; import java.net.URI; import java.net.URISyntaxException; +import java.net.URLDecoder; import java.net.URLEncoder; import java.util.ArrayList; import java.util.List; public class PoolParty implements RecipeInterface { - public static final Logger LOG = LoggerFactory.getLogger(PoolParty.class); - public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final Logger LOG = LoggerFactory.getLogger(PoolParty.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @Override public List fetchMatchingTerms(XdmItem config, String match) { ArrayList terms = Lists.newArrayList(); try { - StringBuilder api = new StringBuilder(Saxon.xpath2string(config, "nde:api", null, OpenSKOS.NAMESPACES)); + String api = Saxon.xpath2string(config, "nde:api", null, OpenSKOS.NAMESPACES); String query = Saxon.xpath2string(config, "nde:query", null, OpenSKOS.NAMESPACES); URLEncoder.encode(query, "UTF-8"); - api.append("?format=application/json&query=").append(query.replace("${match}", match).trim()); + query = URLDecoder.decode(query.replace("${match}", match).trim(), "UTF-8"); CloseableHttpClient client = HttpClients.createDefault(); - HttpGet httpGet = new HttpGet(api.toString()); - CloseableHttpResponse httpResponse = client.execute(httpGet); + HttpPost post = new HttpPost(api); + ArrayList parameters = Lists.newArrayList( + new BasicNameValuePair("format", "application/json"), + new BasicNameValuePair("query", query)); + post.setEntity(new UrlEncodedFormEntity(parameters)); + // HttpGet httpGet = new HttpGet(api.toString()); + CloseableHttpResponse httpResponse = client.execute(post); String theString = entityToString(httpResponse.getEntity()); // LOG.info("response {}", theString); JsonNode jsonNode = OBJECT_MAPPER.readTree(theString); - LOG.info("json results: {}", jsonNode.get("results")); - LOG.info("json bindings: {}", jsonNode.get("results").get("bindings")); ArrayNode results = (ArrayNode) jsonNode.get("results").get("bindings"); results.iterator().forEachRemaining(res -> { TermDTO term = new TermDTO(); From 22784772c6e79e412eaff04cc893a2da8a01d471 Mon Sep 17 00:00:00 2001 From: Martijn Maas Date: Fri, 30 Nov 2018 14:34:32 +0100 Subject: [PATCH 3/4] Make recipe work with RDF XML --- conf/termennetwerk.xml | 2 +- pom.xml | 10 +++ src/main/java/nl/knaw/huc/di/nde/TermDTO.java | 14 ++-- .../nl/knaw/huc/di/nde/recipe/PoolParty.java | 72 ++++++++++++++----- 4 files changed, 72 insertions(+), 26 deletions(-) diff --git a/conf/termennetwerk.xml b/conf/termennetwerk.xml index 116659b..16fc092 100644 --- a/conf/termennetwerk.xml +++ b/conf/termennetwerk.xml @@ -13,7 +13,7 @@ https://data.cultureelerfgoed.nl/PoolParty/sparql/term/id/cht - PREFIX%20skos%3A%3Chttp%3A%2F%2Fwww.w3.org%2F2004%2F02%2Fskos%2Fcore%23%3E%0A%0ASELECT%20%3Furi%20%3FprefLabel%20%3FaltLabel%20%3FhiddenLabel%20%3FscopeNote%0AWHERE%20%7B%0A%20%20%3Furi%20%3Fx%20skos%3AConcept%20.%0A%20%20%3Furi%20skos%3AprefLabel%20%3FprefLabel%20.%0A%20%20OPTIONAL%20%7B%0A%20%20%20%20%3Fsubject%20skos%3AaltLabel%20%3FaltLabel%20.%0A%20%20%20%20%3Fsubject%20skos%3AhiddenLabel%20%3FhiddenLabel%20.%0A%20%20%20%20%3Fsubject%20skos%3AscopeNote%20%3FscopeNote%20.%0A%20%20%7D%0A%20%20FILTER%20%28LANG%28%3FprefLabel%29%20%3D%20%22nl%22%29%20.%0A%20%20FILTER%20%28CONTAINS%28LCASE%28%3FprefLabel%29%2C%20LCASE%28%22${match}%22%29%29%29%0A%7D%0AORDER%20BY%20%3FprefLabel%0ALIMIT%2010%0A + PREFIX%20skos%3A%20%3Chttp%3A%2F%2Fwww.w3.org%2F2004%2F02%2Fskos%2Fcore%23%3E%0ACONSTRUCT%20%7B%0A%20%20%3Furi%20a%20skos%3AConcept%20%3B%0A%20%20%20%20%20%20%20%20%20skos%3AprefLabel%20%3FprefLabel%20%3B%0A%20%20%20%20%20%20%20%20%20skos%3AaltLabel%20%3FaltLabel%20%3B%0A%20%20%20%20%20%20%20%20%20skos%3AhiddenLabel%20%3FhiddenLabel%20%3B%0A%20%20%20%20%20%20%20%20%20skos%3AscopeNote%20%3FscopeNote%20.%0A%7D%0AWHERE%20%7B%0A%20%20%3Furi%20skos%3AprefLabel%20%3FprefLabel%20.%0A%20%20FILTER%20%28LANG%28%3FprefLabel%29%20%3D%20%22nl%22%29%20.%0A%20%20FILTER%20%28CONTAINS%28LCASE%28%3FprefLabel%29%2C%20LCASE%28%22${match}%22%29%29%29%0A%20%20OPTIONAL%20%7B%0A%20%20%20%20%20%3Furi%20skos%3AaltLabel%20%3FaltLabel%20.%0A%20%20%20%20%20%3Furi%20skos%3AhiddenLabel%20%3FhiddenLabel%20.%0A%20%20%20%20%20%3Furi%20skos%3AscopeNote%20%3FscopeNote%20.%0A%20%20%7D%0A%7D%0ALIMIT%201000%0A diff --git a/pom.xml b/pom.xml index 165df77..603b767 100644 --- a/pom.xml +++ b/pom.xml @@ -86,6 +86,16 @@ httpmime 4.5.6 + + org.eclipse.rdf4j + rdf4j-model + 2.4.2 + + + org.eclipse.rdf4j + rdf4j-rio-rdfxml + 2.4.2 + diff --git a/src/main/java/nl/knaw/huc/di/nde/TermDTO.java b/src/main/java/nl/knaw/huc/di/nde/TermDTO.java index cbc5f81..04f5460 100644 --- a/src/main/java/nl/knaw/huc/di/nde/TermDTO.java +++ b/src/main/java/nl/knaw/huc/di/nde/TermDTO.java @@ -1,12 +1,14 @@ package nl.knaw.huc.di.nde; +import com.google.common.collect.Lists; + import java.net.URI; import java.util.List; public class TermDTO { - public URI uri; - public List prefLabel; - public List altLabel; - public List scopeNote; - public List definition; - } + public URI uri; + public List prefLabel = Lists.newArrayList(); + public List altLabel = Lists.newArrayList(); + public List scopeNote = Lists.newArrayList(); + public List definition = Lists.newArrayList(); +} diff --git a/src/main/java/nl/knaw/huc/di/nde/recipe/PoolParty.java b/src/main/java/nl/knaw/huc/di/nde/recipe/PoolParty.java index dafabf5..351f715 100644 --- a/src/main/java/nl/knaw/huc/di/nde/recipe/PoolParty.java +++ b/src/main/java/nl/knaw/huc/di/nde/recipe/PoolParty.java @@ -2,7 +2,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; import com.google.common.collect.Lists; import net.sf.saxon.s9api.SaxonApiException; import net.sf.saxon.s9api.XdmItem; @@ -16,6 +15,11 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.message.BasicNameValuePair; +import org.eclipse.rdf4j.model.Model; +import org.eclipse.rdf4j.model.Statement; +import org.eclipse.rdf4j.rio.RDFFormat; +import org.eclipse.rdf4j.rio.RDFParser; +import org.eclipse.rdf4j.rio.Rio; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,32 +51,62 @@ public List fetchMatchingTerms(XdmItem config, String match) { CloseableHttpClient client = HttpClients.createDefault(); HttpPost post = new HttpPost(api); ArrayList parameters = Lists.newArrayList( - new BasicNameValuePair("format", "application/json"), new BasicNameValuePair("query", query)); post.setEntity(new UrlEncodedFormEntity(parameters)); // HttpGet httpGet = new HttpGet(api.toString()); CloseableHttpResponse httpResponse = client.execute(post); - String theString = entityToString(httpResponse.getEntity()); + // String theString = entityToString(httpResponse.getEntity()); // LOG.info("response {}", theString); - JsonNode jsonNode = OBJECT_MAPPER.readTree(theString); - ArrayNode results = (ArrayNode) jsonNode.get("results").get("bindings"); - results.iterator().forEachRemaining(res -> { - TermDTO term = new TermDTO(); - String uri = res.get("uri").get("value").asText(); - try { - term.uri = new URI(uri); - term.definition = getOptional(res, "hiddenLabel"); - term.altLabel = getOptional(res, "altLabel"); - term.prefLabel = Lists.newArrayList(res.get("prefLabel").get("value").textValue()); - term.scopeNote = getOptional(res, "scopeNote"); - } catch (URISyntaxException e) { - LOG.error("Uri {} not an URI", uri); + + Model model = Rio.parse(httpResponse.getEntity().getContent(), "https://data.cultureelerfgoed.nl/term/id/cht/", + RDFFormat.RDFXML); + String subject = null; + TermDTO term = null; + for (Statement statement : model) { + if(subject == null || !statement.getSubject().stringValue().equals(subject)) { + subject = statement.getSubject().stringValue(); + if(term != null) { + terms.add(term); + } + term = new TermDTO(); + term.uri = new URI(subject); + } + if(statement.getPredicate().getLocalName().equals("prefLabel")) { + term.prefLabel.add(statement.getObject().stringValue()); + } + if(statement.getPredicate().getLocalName().equals("altLabel")) { + term.altLabel.add(statement.getObject().stringValue()); + } + if(statement.getPredicate().getLocalName().equals("hiddenLabel")) { + term.definition.add(statement.getObject().stringValue()); } + if(statement.getPredicate().getLocalName().equals("scopeNote")) { + term.scopeNote.add(statement.getObject().stringValue()); + } + + } + terms.add(term); + - terms.add(term); - }); + // JsonNode jsonNode = OBJECT_MAPPER.readTree(theString); + // ArrayNode results = (ArrayNode) jsonNode.get("results").get("bindings"); + // results.iterator().forEachRemaining(res -> { + // TermDTO term = new TermDTO(); + // String uri = res.get("uri").get("value").asText(); + // try { + // term.uri = new URI(uri); + // term.definition = getOptional(res, "hiddenLabel"); + // term.altLabel = getOptional(res, "altLabel"); + // term.prefLabel = Lists.newArrayList(res.get("prefLabel").get("value").textValue()); + // term.scopeNote = getOptional(res, "scopeNote"); + // } catch (URISyntaxException e) { + // LOG.error("Uri {} not an URI", uri); + // } + // + // terms.add(term); + // }); - } catch (IOException | SaxonApiException e) { + } catch (IOException | SaxonApiException | URISyntaxException e) { LOG.error("Request failed: ", e); } From 555c440913f1b524cd3782b744c1f4dcc95a859c Mon Sep 17 00:00:00 2001 From: Martijn Maas Date: Fri, 30 Nov 2018 14:36:33 +0100 Subject: [PATCH 4/4] Code cleanup --- .../nl/knaw/huc/di/nde/recipe/PoolParty.java | 54 +++---------------- 1 file changed, 6 insertions(+), 48 deletions(-) diff --git a/src/main/java/nl/knaw/huc/di/nde/recipe/PoolParty.java b/src/main/java/nl/knaw/huc/di/nde/recipe/PoolParty.java index 351f715..5f34aea 100644 --- a/src/main/java/nl/knaw/huc/di/nde/recipe/PoolParty.java +++ b/src/main/java/nl/knaw/huc/di/nde/recipe/PoolParty.java @@ -1,14 +1,10 @@ package nl.knaw.huc.di.nde.recipe; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; import net.sf.saxon.s9api.SaxonApiException; import net.sf.saxon.s9api.XdmItem; import nl.knaw.huc.di.nde.TermDTO; import nl.mpi.tla.util.Saxon; -import org.apache.commons.io.IOUtils; -import org.apache.http.HttpEntity; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; @@ -18,13 +14,11 @@ import org.eclipse.rdf4j.model.Model; import org.eclipse.rdf4j.model.Statement; import org.eclipse.rdf4j.rio.RDFFormat; -import org.eclipse.rdf4j.rio.RDFParser; import org.eclipse.rdf4j.rio.Rio; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.io.StringWriter; import java.net.URI; import java.net.URISyntaxException; import java.net.URLDecoder; @@ -35,7 +29,6 @@ public class PoolParty implements RecipeInterface { private static final Logger LOG = LoggerFactory.getLogger(PoolParty.class); - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @Override public List fetchMatchingTerms(XdmItem config, String match) { @@ -53,59 +46,37 @@ public List fetchMatchingTerms(XdmItem config, String match) { ArrayList parameters = Lists.newArrayList( new BasicNameValuePair("query", query)); post.setEntity(new UrlEncodedFormEntity(parameters)); - // HttpGet httpGet = new HttpGet(api.toString()); CloseableHttpResponse httpResponse = client.execute(post); - // String theString = entityToString(httpResponse.getEntity()); - // LOG.info("response {}", theString); Model model = Rio.parse(httpResponse.getEntity().getContent(), "https://data.cultureelerfgoed.nl/term/id/cht/", RDFFormat.RDFXML); String subject = null; TermDTO term = null; for (Statement statement : model) { - if(subject == null || !statement.getSubject().stringValue().equals(subject)) { + if (subject == null || !statement.getSubject().stringValue().equals(subject)) { subject = statement.getSubject().stringValue(); - if(term != null) { + if (term != null) { terms.add(term); } term = new TermDTO(); term.uri = new URI(subject); } - if(statement.getPredicate().getLocalName().equals("prefLabel")) { + if (statement.getPredicate().getLocalName().equals("prefLabel")) { term.prefLabel.add(statement.getObject().stringValue()); } - if(statement.getPredicate().getLocalName().equals("altLabel")) { + if (statement.getPredicate().getLocalName().equals("altLabel")) { term.altLabel.add(statement.getObject().stringValue()); } - if(statement.getPredicate().getLocalName().equals("hiddenLabel")) { + if (statement.getPredicate().getLocalName().equals("hiddenLabel")) { term.definition.add(statement.getObject().stringValue()); } - if(statement.getPredicate().getLocalName().equals("scopeNote")) { + if (statement.getPredicate().getLocalName().equals("scopeNote")) { term.scopeNote.add(statement.getObject().stringValue()); } } terms.add(term); - - // JsonNode jsonNode = OBJECT_MAPPER.readTree(theString); - // ArrayNode results = (ArrayNode) jsonNode.get("results").get("bindings"); - // results.iterator().forEachRemaining(res -> { - // TermDTO term = new TermDTO(); - // String uri = res.get("uri").get("value").asText(); - // try { - // term.uri = new URI(uri); - // term.definition = getOptional(res, "hiddenLabel"); - // term.altLabel = getOptional(res, "altLabel"); - // term.prefLabel = Lists.newArrayList(res.get("prefLabel").get("value").textValue()); - // term.scopeNote = getOptional(res, "scopeNote"); - // } catch (URISyntaxException e) { - // LOG.error("Uri {} not an URI", uri); - // } - // - // terms.add(term); - // }); - } catch (IOException | SaxonApiException | URISyntaxException e) { LOG.error("Request failed: ", e); } @@ -113,17 +84,4 @@ public List fetchMatchingTerms(XdmItem config, String match) { return terms; } - - private ArrayList getOptional(JsonNode res, String field) { - if (res.has(field)) { - return Lists.newArrayList(res.get(field).get("value").asText()); - } - return Lists.newArrayList(); - } - - private String entityToString(HttpEntity responseEntity) throws IOException { - StringWriter writer = new StringWriter(); - IOUtils.copy(responseEntity.getContent(), writer, "UTF-8"); - return writer.toString(); - } }