Skip to content

Commit

Permalink
Fixed the cache insanity
Browse files Browse the repository at this point in the history
  • Loading branch information
@romanchyla
romanchyla committed Jun 17, 2014
1 parent 5a5314e commit 5befc41
Show file tree
Hide file tree
Showing 9 changed files with 83 additions and 22 deletions.
2 changes: 1 addition & 1 deletion .classpath
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@
<classpathentry kind="lib" path="build/solrjars-extracted/lucene-memory-4.8-SNAPSHOT.jar"/>
<classpathentry kind="lib" path="build/solrjars-extracted/lucene-misc-4.8-SNAPSHOT.jar"/>
<classpathentry kind="lib" path="build/solrjars-extracted/lucene-queries-4.8-SNAPSHOT.jar"/>
<classpathentry kind="lib" path="build/solrjars-extracted/lucene-queryparser-4.8-SNAPSHOT.jar"/>
<classpathentry kind="lib" path="build/solrjars-extracted/lucene-queryparser-4.8-SNAPSHOT.jar" sourcepath="build/solr-download/apache-solr-4.0.0-SVN/lucene/queryparser/src/java"/>
<classpathentry kind="lib" path="build/solrjars-extracted/lucene-replicator-4.8-SNAPSHOT.jar"/>
<classpathentry kind="lib" path="build/solrjars-extracted/lucene-sandbox-4.8-SNAPSHOT.jar"/>
<classpathentry kind="lib" path="build/solrjars-extracted/lucene-spatial-4.8-SNAPSHOT.jar"/>
Expand Down
2 changes: 2 additions & 0 deletions contrib/adsabs/src/java/org/apache/solr/handler/batch/BatchProviderDumpIndexFields.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,8 @@ public void collect(int i) throws IOException {
tokens.add(fType.indexedToReadable(termAtt.toString()));
}
}

buffer.close();
}
}
}
Expand Down
50 changes: 49 additions & 1 deletion contrib/adsabs/src/java/org/apache/solr/search/CitationLRUCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,12 @@
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocTermOrds;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.FieldCache.CacheEntry;
import org.apache.lucene.search.FieldCache.Ints;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
Expand Down Expand Up @@ -732,9 +734,11 @@ private void unInvertedTheDamnThing(AtomicReader reader, Map<String,
}
}


// load single valued ids
for (String idField: fields.get("textFields")) {
BinaryDocValues idMapping = FieldCache.DEFAULT.getTerms(reader, idField, false); // XXX:rca - should we use 'true'?
BinaryDocValues idMapping = getCacheReuseExisting(reader, idField);

Integer i = 0;
BytesRef ret = new BytesRef();
while(i < reader.maxDoc()) {
Expand Down Expand Up @@ -766,6 +770,50 @@ private void unInvertedTheDamnThing(AtomicReader reader, Map<String,

}

private BinaryDocValues getCacheReuseExisting(AtomicReader reader, String idField) throws IOException {

Boolean sorted = false;
if (idField.indexOf(':') > -1) {
String[] parts = idField.split(":");
idField = parts[0];
if (parts[1].indexOf("sort") > -1)
sorted = true;
}

// first try discover if there exists a cache already that we can reuse
CacheEntry[] caches = FieldCache.DEFAULT.getCacheEntries();
if (caches != null) {
for (int i=0; i < caches.length; i++) {

CacheEntry c = caches[i];
String key = c.getFieldName();
Object readerKey = c.getReaderKey();

if (idField.equals(key) && readerKey.toString().equals(reader.getCoreCacheKey().toString())) {
Object v = c.getValue();
if (v instanceof BinaryDocValues) {
return (BinaryDocValues) v;
}
}

}
}

BinaryDocValues idMapping;

// because sorting components will create the cache anyway; we can avoid duplicating data
// if we create a cache duplicate, the tests will complain about cache insanity (and rightly so)
if (sorted) {
//System.out.println("creating new sorted: " + idField);
idMapping = FieldCache.DEFAULT.getTermsIndex(reader, idField);
}
else {
//System.out.println("creating new: " + idField);
idMapping = FieldCache.DEFAULT.getTerms(reader, idField, false); // XXX:rca - should we use 'true'?
}
return idMapping;

}
//////////////////////// SolrInfoMBeans methods //////////////////////


Expand Down
13 changes: 7 additions & 6 deletions contrib/adsabs/src/test/org/adsabs/TestAdsabsIndexingSearching.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,19 +45,20 @@ public class TestAdsabsIndexingSearching extends MontySolrQueryTestCase {

@BeforeClass
public static void beforeClass() throws Exception {
makeResourcesVisible(Thread.currentThread().getContextClassLoader(), new String[] {
MontySolrSetup.getMontySolrHome() + "/contrib/examples/adsabs/solr/collection1/conf",
MontySolrSetup.getSolrHome() + "/example/solr/collection1/conf"
});

System.setProperty("solr.allow.unsafe.resourceloading", "true");
schemaString = MontySolrSetup.getMontySolrHome()
+ "/contrib/examples/adsabs/solr/collection1/conf/schema.xml";

configString = MontySolrSetup.getMontySolrHome()
+ "/contrib/examples/adsabs/solr/collection1/conf/solrconfig.xml";

/*makeResourcesVisible(???,
new String[] {MontySolrSetup.getMontySolrHome() + "/contrib/examples/adsabs/solr/collection1/conf",
MontySolrSetup.getSolrHome() + "/example/solr/collection1/conf"
});*/

initCore(configString, schemaString);
initCore(configString, schemaString, MontySolrSetup.getSolrHome()
+ "/example/solr");
}


Expand Down
9 changes: 8 additions & 1 deletion contrib/adsabs/src/test/org/apache/lucene/search/TestCitationsSearch.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,11 @@ public class TestCitationsSearch extends MontySolrAbstractTestCase {

@BeforeClass
public static void beforeClass() throws Exception {
makeResourcesVisible(Thread.currentThread().getContextClassLoader(), new String[] {
MontySolrSetup.getMontySolrHome() + "/contrib/examples/adsabs/solr/collection1/conf",
MontySolrSetup.getSolrHome() + "/example/solr/collection1/conf"
});

System.setProperty("solr.allow.unsafe.resourceloading", "true");
schemaString = MontySolrSetup.getMontySolrHome()
+ "/contrib/adsabs/src/test-files/solr/collection1/conf/"
Expand All @@ -42,7 +47,9 @@ public static void beforeClass() throws Exception {
configString = MontySolrSetup.getMontySolrHome()
+ "/contrib/adsabs/src/test-files/solr/collection1/conf/"
+ "citation-cache-solrconfig.xml";
initCore(configString, schemaString);

initCore(configString, schemaString, MontySolrSetup.getSolrHome()
+ "/example/solr");
}


Expand Down
4 changes: 3 additions & 1 deletion contrib/adsabs/src/test/org/apache/lucene/search/TestSecondOrderQueryTypesAds.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,9 @@ public static void beforeClass() throws Exception {
configString = MontySolrSetup.getMontySolrHome()
+ "/contrib/adsabs/src/test-files/solr/collection1/conf/"
+ "citation-cache-solrconfig.xml";
initCore(configString, schemaString);

initCore(configString, schemaString, MontySolrSetup.getSolrHome()
+ "/example/solr");
}

private SolrQueryRequest tempReq;
Expand Down
15 changes: 8 additions & 7 deletions contrib/adsabs/src/test/org/apache/solr/analysis/TestAdsabsTypeAffiliationText.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,22 +32,23 @@ public class TestAdsabsTypeAffiliationText extends MontySolrQueryTestCase {

@BeforeClass
public static void beforeClass() throws Exception {

makeResourcesVisible(Thread.currentThread().getContextClassLoader(), new String[] {
MontySolrSetup.getMontySolrHome() + "/contrib/examples/adsabs/solr/collection1/conf",
MontySolrSetup.getSolrHome() + "/example/solr/collection1/conf"
});

System.setProperty("solr.allow.unsafe.resourceloading", "true");

/*
makeResourcesVisible(this.solrConfig.getResourceLoader(),
new String[] {MontySolrSetup.getMontySolrHome() + "/contrib/examples/adsabs/solr/collection1/conf",
MontySolrSetup.getSolrHome() + "/example/solr/collection1/conf"
});
*/

schemaString = MontySolrSetup.getMontySolrHome()
+ "/contrib/examples/adsabs/solr/collection1/conf/schema.xml";

configString = MontySolrSetup.getMontySolrHome()
+ "/contrib/examples/adsabs/solr/collection1/conf/solrconfig.xml";

initCore(configString, schemaString);
initCore(configString, schemaString, MontySolrSetup.getSolrHome()
+ "/example/solr");
}


Expand Down
8 changes: 4 additions & 4 deletions contrib/adsabs/src/test/org/apache/solr/analysis/author/TestAdsabsTypeAuthorParsing.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,10 +83,10 @@ public class TestAdsabsTypeAuthorParsing extends MontySolrQueryTestCase {
@BeforeClass
public static void beforeClass() throws Exception {

makeResourcesVisible(Thread.currentThread().getContextClassLoader(),
new String[] {MontySolrSetup.getMontySolrHome() + "/contrib/examples/adsabs/solr/collection1/conf",
MontySolrSetup.getSolrHome() + "/example/solr/collection1/conf"
});
makeResourcesVisible(Thread.currentThread().getContextClassLoader(), new String[] {
MontySolrSetup.getMontySolrHome() + "/contrib/examples/adsabs/solr/collection1/conf",
MontySolrSetup.getSolrHome() + "/example/solr/collection1/conf"
});

System.setProperty("solr.allow.unsafe.resourceloading", "true");
schemaString = getSchemaFile();
Expand Down
2 changes: 1 addition & 1 deletion contrib/examples/adsabs/solr/collection1/conf/solrconfig.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -554,7 +554,7 @@
initialSize="1024"
autowarmCount="512"
regenerator="solr.CitationLRUCache$SimpleRegenerator"
identifierFields="bibcode,alternate_bibcode"
identifierFields="bibcode:sorted,alternate_bibcode"
citationFields="citation"
/>

Expand Down

0 comments on commit 5befc41

Please sign in to comment.