diff --git a/.classpath b/.classpath index a1be320..80b3569 100644 --- a/.classpath +++ b/.classpath @@ -19,26 +19,17 @@ - - - - - - - - - @@ -65,5 +56,26 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/module/MOD-INF/lib/apache-any23-api-0.9.0.jar b/module/MOD-INF/lib/apache-any23-api-0.8.0.jar similarity index 81% rename from module/MOD-INF/lib/apache-any23-api-0.9.0.jar rename to module/MOD-INF/lib/apache-any23-api-0.8.0.jar index f873585..91b2658 100644 Binary files a/module/MOD-INF/lib/apache-any23-api-0.9.0.jar and b/module/MOD-INF/lib/apache-any23-api-0.8.0.jar differ diff --git a/module/MOD-INF/lib/apache-any23-core-0.9.0.jar b/module/MOD-INF/lib/apache-any23-core-0.8.0.jar similarity index 70% rename from module/MOD-INF/lib/apache-any23-core-0.9.0.jar rename to module/MOD-INF/lib/apache-any23-core-0.8.0.jar index c91a726..7255f44 100644 Binary files a/module/MOD-INF/lib/apache-any23-core-0.9.0.jar and b/module/MOD-INF/lib/apache-any23-core-0.8.0.jar differ diff --git a/module/MOD-INF/lib/apache-any23-encoding-0.9.0.jar b/module/MOD-INF/lib/apache-any23-encoding-0.8.0.jar similarity index 66% rename from module/MOD-INF/lib/apache-any23-encoding-0.9.0.jar rename to module/MOD-INF/lib/apache-any23-encoding-0.8.0.jar index cbbb522..99ab82b 100644 Binary files a/module/MOD-INF/lib/apache-any23-encoding-0.9.0.jar and b/module/MOD-INF/lib/apache-any23-encoding-0.8.0.jar differ diff --git a/module/MOD-INF/lib/apache-any23-mime-0.9.0.jar b/module/MOD-INF/lib/apache-any23-mime-0.8.0.jar similarity index 88% rename from module/MOD-INF/lib/apache-any23-mime-0.9.0.jar rename to module/MOD-INF/lib/apache-any23-mime-0.8.0.jar index 798414a..ee563fa 100644 Binary files a/module/MOD-INF/lib/apache-any23-mime-0.9.0.jar and b/module/MOD-INF/lib/apache-any23-mime-0.8.0.jar differ diff --git a/module/MOD-INF/lib/commons-codec-1.6.jar b/module/MOD-INF/lib/commons-codec-1.6.jar new file mode 100644 index 0000000..ee1bc49 Binary files /dev/null and b/module/MOD-INF/lib/commons-codec-1.6.jar differ diff --git a/module/MOD-INF/lib/jcl-over-slf4j-1.6.4.jar b/module/MOD-INF/lib/jcl-over-slf4j-1.6.4.jar new file mode 100644 index 0000000..3d0bb1f Binary files /dev/null and b/module/MOD-INF/lib/jcl-over-slf4j-1.6.4.jar differ diff --git a/module/MOD-INF/lib/jena-arq-2.11.1.jar b/module/MOD-INF/lib/jena-arq-2.11.1.jar new file mode 100644 index 0000000..a7e0fdb Binary files /dev/null and b/module/MOD-INF/lib/jena-arq-2.11.1.jar differ diff --git a/module/MOD-INF/lib/jena-arq-2.9.0-incubating.jar b/module/MOD-INF/lib/jena-arq-2.9.0-incubating.jar deleted file mode 100644 index d4ac9a3..0000000 Binary files a/module/MOD-INF/lib/jena-arq-2.9.0-incubating.jar and /dev/null differ diff --git a/module/MOD-INF/lib/jena-core-2.11.1.jar b/module/MOD-INF/lib/jena-core-2.11.1.jar new file mode 100644 index 0000000..a8a32a7 Binary files /dev/null and b/module/MOD-INF/lib/jena-core-2.11.1.jar differ diff --git a/module/MOD-INF/lib/jena-core-2.7.0-incubating.jar b/module/MOD-INF/lib/jena-core-2.7.0-incubating.jar deleted file mode 100644 index b0068f3..0000000 Binary files a/module/MOD-INF/lib/jena-core-2.7.0-incubating.jar and /dev/null differ diff --git a/module/MOD-INF/lib/jena-iri-0.9.0-incubating.jar b/module/MOD-INF/lib/jena-iri-0.9.0-incubating.jar deleted file mode 100644 index 23830b7..0000000 Binary files a/module/MOD-INF/lib/jena-iri-0.9.0-incubating.jar and /dev/null differ diff --git a/module/MOD-INF/lib/jena-iri-1.0.1.jar b/module/MOD-INF/lib/jena-iri-1.0.1.jar new file mode 100644 index 0000000..dbfc63d Binary files /dev/null and b/module/MOD-INF/lib/jena-iri-1.0.1.jar differ diff --git a/module/MOD-INF/lib/jena-larq-1.0.0-incubating.jar b/module/MOD-INF/lib/jena-larq-1.0.0-incubating.jar deleted file mode 100644 index 11ac1ed..0000000 Binary files a/module/MOD-INF/lib/jena-larq-1.0.0-incubating.jar and /dev/null differ diff --git a/module/MOD-INF/lib/jena-sdb-1.4.1.jar b/module/MOD-INF/lib/jena-sdb-1.4.1.jar new file mode 100644 index 0000000..6d99119 Binary files /dev/null and b/module/MOD-INF/lib/jena-sdb-1.4.1.jar differ diff --git a/module/MOD-INF/lib/jena-tdb-1.0.1.jar b/module/MOD-INF/lib/jena-tdb-1.0.1.jar new file mode 100644 index 0000000..9654339 Binary files /dev/null and b/module/MOD-INF/lib/jena-tdb-1.0.1.jar differ diff --git a/module/MOD-INF/lib/jena-text-1.0.1.jar b/module/MOD-INF/lib/jena-text-1.0.1.jar new file mode 100644 index 0000000..ea109ef Binary files /dev/null and b/module/MOD-INF/lib/jena-text-1.0.1.jar differ diff --git a/module/MOD-INF/lib/lucene-analyzers-common-4.3.1.jar b/module/MOD-INF/lib/lucene-analyzers-common-4.3.1.jar new file mode 100644 index 0000000..e44da43 Binary files /dev/null and b/module/MOD-INF/lib/lucene-analyzers-common-4.3.1.jar differ diff --git a/module/MOD-INF/lib/lucene-core-3.6.0.jar b/module/MOD-INF/lib/lucene-core-3.6.0.jar deleted file mode 100644 index b973f1f..0000000 Binary files a/module/MOD-INF/lib/lucene-core-3.6.0.jar and /dev/null differ diff --git a/module/MOD-INF/lib/lucene-core-4.3.1.jar b/module/MOD-INF/lib/lucene-core-4.3.1.jar new file mode 100644 index 0000000..9a747f6 Binary files /dev/null and b/module/MOD-INF/lib/lucene-core-4.3.1.jar differ diff --git a/module/MOD-INF/lib/lucene-queryparser-4.3.1.jar b/module/MOD-INF/lib/lucene-queryparser-4.3.1.jar new file mode 100644 index 0000000..6443c37 Binary files /dev/null and b/module/MOD-INF/lib/lucene-queryparser-4.3.1.jar differ diff --git a/module/MOD-INF/lib/slf4j-api-1.7.5.jar b/module/MOD-INF/lib/slf4j-api-1.7.5.jar new file mode 100644 index 0000000..8766455 Binary files /dev/null and b/module/MOD-INF/lib/slf4j-api-1.7.5.jar differ diff --git a/module/MOD-INF/lib/slf4j-log4j12-1.7.5.jar b/module/MOD-INF/lib/slf4j-log4j12-1.7.5.jar new file mode 100644 index 0000000..afce5c2 Binary files /dev/null and b/module/MOD-INF/lib/slf4j-log4j12-1.7.5.jar differ diff --git a/module/MOD-INF/lib/solr-solrj-4.3.1.jar b/module/MOD-INF/lib/solr-solrj-4.3.1.jar new file mode 100644 index 0000000..a4e3bee Binary files /dev/null and b/module/MOD-INF/lib/solr-solrj-4.3.1.jar differ diff --git a/module/scripts/rdf-schema-alignment.js b/module/scripts/rdf-schema-alignment.js index c4e5a28..3ed7908 100644 --- a/module/scripts/rdf-schema-alignment.js +++ b/module/scripts/rdf-schema-alignment.js @@ -150,13 +150,13 @@ RdfSchemaAlignmentDialog.prototype._renderBody = function(body) { var self = this; $("#rdf-schema-alignment-tabs").tabs({ - activate:function(evt,tabs){ - if(tabs.newPanel.index()===2){ + select:function(evt,ui){ + if(ui.index===1){ self._previewRdf(); } } }); - // $("#rdf-schema-alignment-tabs-preview").css("display", ""); + $("#rdf-schema-alignment-tabs-preview").css("display", ""); // $("#rdf-schema-alignment-tabs-vocabulary-manager").css("display", ""); this._canvas = $(".schema-alignment-dialog-canvas"); diff --git a/src/org/deri/grefine/rdf/vocab/IVocabularySearcher.java b/src/org/deri/grefine/rdf/vocab/IVocabularySearcher.java index eee27e8..54d3004 100644 --- a/src/org/deri/grefine/rdf/vocab/IVocabularySearcher.java +++ b/src/org/deri/grefine/rdf/vocab/IVocabularySearcher.java @@ -5,7 +5,6 @@ import java.util.Set; import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.queryParser.ParseException; import org.openrdf.repository.Repository; @@ -32,9 +31,9 @@ public interface IVocabularySearcher { public void importAndIndexVocabulary(String name, String uri, String fetchUrl,String projectId,VocabularyImporter importer) throws VocabularyImportException, VocabularyIndexException, PrefixExistException, CorruptIndexException, IOException; public void importAndIndexVocabulary(String name, String uri, Repository repository, String projectId,VocabularyImporter importer) throws VocabularyImportException, VocabularyIndexException, PrefixExistException, CorruptIndexException, IOException; - public List searchClasses(String str, String projectId) throws ParseException,IOException; + public List searchClasses(String str, String projectId) throws IOException; - public List searchProperties(String str, String projectId) throws ParseException,IOException; + public List searchProperties(String str, String projectId) throws IOException; public void deleteTermsOfVocabs(Set toRemove,String projectId) throws CorruptIndexException, IOException; public void deleteTermsOfVocab(String vocabName, String projectId) throws CorruptIndexException, IOException; diff --git a/src/org/deri/grefine/rdf/vocab/imp/VocabularySearcher.java b/src/org/deri/grefine/rdf/vocab/imp/VocabularySearcher.java index 7d25925..ccef862 100644 --- a/src/org/deri/grefine/rdf/vocab/imp/VocabularySearcher.java +++ b/src/org/deri/grefine/rdf/vocab/imp/VocabularySearcher.java @@ -10,18 +10,18 @@ import java.util.Set; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.SimpleAnalyzer; +import org.apache.lucene.analysis.core.SimpleAnalyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.Fieldable; import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.Term; -import org.apache.lucene.queryParser.ParseException; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.IndexSearcher; @@ -58,17 +58,18 @@ public class VocabularySearcher implements IVocabularySearcher { private IndexWriter writer; private IndexSearcher searcher; + private IndexReader r; private Directory _directory; public VocabularySearcher(File dir) throws IOException { _directory = new SimpleFSDirectory(new File(dir, "luceneIndex")); - Analyzer a = new SimpleAnalyzer(Version.LUCENE_36); - IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_36,a); + Analyzer a = new SimpleAnalyzer(Version.LUCENE_43); + IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43,a); writer = new IndexWriter(_directory,conf); writer.commit(); - IndexReader r = IndexReader.open(_directory); + r = DirectoryReader.open(_directory); searcher = new IndexSearcher(r); } @@ -98,7 +99,7 @@ public void importAndIndexVocabulary(String name, String uri, Repository reposit @Override public List searchClasses(String str, String projectId) - throws ParseException, IOException { + throws IOException { Query query = prepareQuery(str, CLASS_TYPE, projectId); TopDocs docs = searcher.search(query, getMaxDoc()); return prepareSearchResults(docs); @@ -106,7 +107,7 @@ public List searchClasses(String str, String projectId) @Override public List searchProperties(String str, String projectId) - throws ParseException, IOException { + throws IOException { Query query = prepareQuery(str, PROPERTY_TYPE, projectId); TopDocs docs = searcher.search(query, getMaxDoc()); return prepareSearchResults(docs); @@ -136,8 +137,9 @@ public void update() throws CorruptIndexException, IOException { writer.commit(); // TODO this shouldn't be required but it is not working without it... // check - searcher.close(); - searcher = new IndexSearcher(IndexReader.open(_directory)); + r.close(); + r = IndexReader.open(_directory); + searcher = new IndexSearcher(r); } @Override @@ -220,7 +222,7 @@ private void indexRdfNode(RDFNode node, String type, String projectId) } private Query prepareQuery(String s, String type, String projectId) - throws ParseException, IOException { + throws IOException { BooleanQuery q1 = new BooleanQuery(); // q1.add(new TermQuery(new // Term("projectId",GLOBAL_VOCABULARY_PLACE_HOLDER)), Occur.SHOULD); @@ -357,7 +359,7 @@ private void addDocumentsToProject(TopDocs docs,String projectId) throws Corrupt Document newdoc = new Document(); Iterator fieldsIter = doc.getFields().iterator(); while(fieldsIter.hasNext()){ - newdoc.add((Fieldable)fieldsIter.next()); + newdoc.add((IndexableField)fieldsIter.next()); } newdoc.removeField("projectId"); newdoc.add(new Field("projectId",projectId,Field.Store.YES,Field.Index.NOT_ANALYZED)); @@ -407,6 +409,6 @@ private void deletePrefixesOfProjectId(String projectId, Set toDelete) t } private int getMaxDoc() throws IOException { - return searcher.maxDoc() > 0 ? searcher.maxDoc() : 100000; + return r.maxDoc() > 0 ? r.maxDoc() : 100000; } } diff --git a/src/org/deri/grefine/reconcile/rdf/executors/DumpQueryExecutor.java b/src/org/deri/grefine/reconcile/rdf/executors/DumpQueryExecutor.java index 8bafab2..ddb35d0 100644 --- a/src/org/deri/grefine/reconcile/rdf/executors/DumpQueryExecutor.java +++ b/src/org/deri/grefine/reconcile/rdf/executors/DumpQueryExecutor.java @@ -4,21 +4,24 @@ import java.io.FileOutputStream; import java.io.IOException; -import org.apache.jena.larq.IndexBuilderString; -import org.apache.jena.larq.IndexLARQ; -import org.apache.jena.larq.LARQ; +import org.apache.jena.query.text.EntityDefinition; +import org.apache.jena.query.text.TextDatasetFactory; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; import org.json.JSONException; import org.json.JSONWriter; +import com.hp.hpl.jena.query.Dataset; +import com.hp.hpl.jena.query.DatasetFactory; import com.hp.hpl.jena.query.Query; import com.hp.hpl.jena.query.QueryExecution; import com.hp.hpl.jena.query.QueryExecutionFactory; import com.hp.hpl.jena.query.QueryFactory; +import com.hp.hpl.jena.query.ReadWrite; import com.hp.hpl.jena.query.ResultSet; import com.hp.hpl.jena.query.Syntax; import com.hp.hpl.jena.rdf.model.Model; import com.hp.hpl.jena.rdf.model.ModelFactory; -import com.hp.hpl.jena.rdf.model.Property; /** * @author fadmaa @@ -29,8 +32,7 @@ */ public class DumpQueryExecutor implements QueryExecutor { - private Model model; - private IndexLARQ index; + private Dataset index; private boolean loaded = false; //property used for index/search (only if one property is used) private String propertyUri; @@ -63,18 +65,18 @@ public DumpQueryExecutor(Model m){ public DumpQueryExecutor(Model m, String propertyUri, boolean ngramIndex,int minGram, int maxGram){ loaded = true; - this.model = m; this.propertyUri = propertyUri; - IndexBuilderString larqBuilder; - if(propertyUri != null){ - Property p = model.getProperty(propertyUri); - larqBuilder= new IndexBuilderString(p) ; - }else{ - larqBuilder= new IndexBuilderString() ; - } - larqBuilder.indexStatements(model.listStatements()) ; - larqBuilder.closeWriter() ; - this.index = larqBuilder.getIndex() ; + + Dataset ds1 = DatasetFactory.createMem(); + EntityDefinition entDef = new EntityDefinition("uri", "text",m.getResource(propertyUri)) ; + + // Lucene, in memory. + Directory dir = new RAMDirectory(); + + // Join together into a dataset + this.index = TextDatasetFactory.createLucene(ds1, dir, entDef) ; + this.index.getDefaultModel().add(m); + //this.index.commit(); } @Override @@ -82,9 +84,9 @@ public ResultSet sparql(String sparql) { if(!loaded){ throw new RuntimeException("Model is not loaded"); } + //this.index.begin(ReadWrite.READ) ; Query query = QueryFactory.create(sparql, Syntax.syntaxSPARQL_11); - QueryExecution qExec = QueryExecutionFactory.create(query, model); - LARQ.setDefaultIndex(qExec.getContext(), index); + QueryExecution qExec = QueryExecutionFactory.create(query, this.index); ResultSet result = qExec.execSelect(); return result; } @@ -101,7 +103,8 @@ public void write(JSONWriter writer) throws JSONException { } public void dispose(){ - model = null; //free the memory used for the model + this.index.close(); + this.index = null; //free the memory used for the model } public synchronized void initialize(FileInputStream in) { @@ -110,22 +113,19 @@ public synchronized void initialize(FileInputStream in) { } loaded = true; // -- Read and index all literal strings. - IndexBuilderString larqBuilder; - model = ModelFactory.createDefaultModel(); + Model model = ModelFactory.createDefaultModel(); model.read(in, null,"TTL"); - if(propertyUri==null){ - larqBuilder = new IndexBuilderString() ; - }else{ - Property p = model.getProperty(propertyUri); - larqBuilder = new IndexBuilderString(p); - } - - larqBuilder.indexStatements(model.listStatements()) ; - // -- Finish indexing - larqBuilder.closeWriter() ; + + Dataset ds1 = DatasetFactory.createMem(); + EntityDefinition entDef = new EntityDefinition("uri", "text",model.getResource(propertyUri)) ; - // -- Create the access index - index = larqBuilder.getIndex() ; + // Lucene, in memory. + Directory dir = new RAMDirectory(); + + // Join together into a dataset + this.index = TextDatasetFactory.createLucene(ds1, dir, entDef) ; + this.index.getDefaultModel().add(model); + this.index.commit(); } private static final int DEFAULT_MIN_NGRAM = 3; @@ -133,7 +133,7 @@ public synchronized void initialize(FileInputStream in) { @Override public void save(String serviceId, FileOutputStream out) throws IOException { - model.write(out, "TTL"); + this.index.getDefaultModel().write(out, "TTL"); out.close(); } } \ No newline at end of file diff --git a/src/org/deri/grefine/reconcile/rdf/factories/LarqSparqlQueryFactory.java b/src/org/deri/grefine/reconcile/rdf/factories/LarqSparqlQueryFactory.java index e7cc923..0635edc 100644 --- a/src/org/deri/grefine/reconcile/rdf/factories/LarqSparqlQueryFactory.java +++ b/src/org/deri/grefine/reconcile/rdf/factories/LarqSparqlQueryFactory.java @@ -3,7 +3,7 @@ import java.util.ArrayList; import java.util.List; -import org.apache.lucene.queryParser.QueryParser; +import org.apache.lucene.queryparser.classic.QueryParser; import org.deri.grefine.reconcile.model.ReconciliationRequest; import org.deri.grefine.reconcile.model.SearchResultItem; import org.deri.grefine.reconcile.model.ReconciliationRequestContext.PropertyContext; @@ -70,6 +70,7 @@ private String getReconciliationSparqlQuery(String queryTemplate, ImmutableList< //the answers are ordered according to their scores descendingly. thus we need to pick only the *first* request.getLimit() *unique* entity answer int calculatedLimit = Math.max(searchPropertyUris.size(),1) * limit; return queryTemplate.replace("[[QUERY]]", escapedQuery) + .replace(labelPlaceHolder, labelFilter) .replace(labelPlaceHolder, labelFilter) .replace("[[TYPE_FILTER]]", typesFilter) .replace("[[CONTEXT_FILTER]]", contextFilter) @@ -233,13 +234,13 @@ public double getScore() { "ORDER BY DESC(?score1) LIMIT [[LIMIT]]"; private static final String SINGLE_LABEL_PROPERTY_RECONCILE_QUERY_TEMPLATE = "PREFIX rdfs: " + - "PREFIX pf: " + + "PREFIX text: " + "PREFIX rdf: "+ - "SELECT ?entity ?label (MAX(?score) AS ?score1) " + - "WHERE" + - "{" + - "?entity <[[LABEL_PROPERTY_URI]]> ?label." + - "(?label ?score) pf:textMatch '[[QUERY]]'. " + + "SELECT ?entity ?label " + + "WHERE " + + "{ " + + "?entity text:query (<[[LABEL_PROPERTY_URI]]> '[[QUERY]]' [[LIMIT]]) . " + + "?entity <[[LABEL_PROPERTY_URI]]> ?label ." + "[[TYPE_FILTER]]" + "[[CONTEXT_FILTER]]" + "}GROUP BY ?entity ?label " +