Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

making storage engines pluggable #27

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

making storage engines pluggable #27

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2912e63
making storage engines pluggable
dbuthay Feb 14, 2012
117bd47
changing help for --storage cli argument
dbuthay Feb 14, 2012
25908d8
fixing unit tests .. they all pass now
dbuthay Feb 15, 2012
106f36f
Merge branch 'master' into pluggable_storage
dbuthay Feb 24, 2012
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 68 additions & 41 deletions src/main/java/com/flaptor/indextank/index/IndexEngine.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@
import com.flaptor.indextank.search.SnippetSearcher;
import com.flaptor.indextank.search.TrafficLimitingSearcher;
import com.flaptor.indextank.storage.alternatives.DocumentStorage;
import com.flaptor.indextank.storage.alternatives.DocumentStorageFactory;
import com.flaptor.indextank.suggest.DidYouMeanSuggestor;
import com.flaptor.indextank.suggest.NoSuggestor;
import com.flaptor.indextank.suggest.QuerySuggestor;
Expand Down Expand Up @@ -112,19 +113,15 @@ public class IndexEngine {

private static final int DEFAULT_BASE_PORT = 7910;
private static final int DEFAULT_RTI_SIZE = 1000;
private static final int DEFAULT_BDB_CACHE = 100;

public static enum SuggestValues { NO, QUERIES, DOCUMENTS};
public static enum StorageValues { NO, BDB, RAM, CASSANDRA };

public IndexEngine( File baseDir,
int basePort,
int rtiSize,
boolean load,
int boostsSize,
SuggestValues suggest,
StorageValues storageValue,
int bdbCache,
String functions,
boolean facets,
String indexCode,
Expand All @@ -136,8 +133,6 @@ public IndexEngine( File baseDir,
load,
boostsSize,
suggest,
storageValue,
bdbCache,
functions,
facets,
indexCode,
Expand All @@ -152,8 +147,6 @@ public IndexEngine( File baseDir,
boolean load,
int boostsSize,
SuggestValues suggest,
StorageValues storageValue,
int bdbCache,
String functions,
boolean facets,
String indexCode,
Expand Down Expand Up @@ -276,18 +269,9 @@ public IndexEngine( File baseDir,
logger.info("Index recovery configuration set to recover index from simpleDB");
this.recoveryStorage = IndexRecoverer.IndexStorageValue.SIMPLEDB;
}

switch (storageValue) {
case RAM:
storage = new InMemoryStorage(baseDir, load);
logger.info("Using in-memory storage");
break;
case NO:
storage = null;
logger.info("NOT Using storage");
break;
}


storage = buildStorage(configuration);

promoter = new BasicPromoter(baseDir, load);
searcher = new Blender(lsi, rti, suggestor, promoter, boostsManager);
indexer = new Dealer(lsi, rti, suggestor, boostsManager, rtiSize, promoter, functionsManager);
Expand Down Expand Up @@ -326,6 +310,54 @@ private Analyzer buildAnalyzer(Map<Object, Object> configuration) {
return analyzer;
};

public DocumentStorage buildStorage(Map<?, ?> configuration) {

DocumentStorage storage = null;

// parse storage from configuration
String storageClassStr = (String) configuration.get("storage");
if (null == storageClassStr || "".equals(storageClassStr.trim()) ) {
logger.info("NOT Using storage");
} else {
try {
// it is either a factory, or a concrete class
Class storageClass = Class.forName(storageClassStr);

if (DocumentStorageFactory.class.isAssignableFrom(storageClass)) {
// if it was a factory ..
logger.info("got a DocumentStorageFactory: " + storageClass);
DocumentStorageFactory storageFactory = (DocumentStorageFactory) storageClass.newInstance();
Map<?, ?> storageConfig = (Map<?, ?>)configuration.get("storage_config");
if (null == storageConfig) {
logger.warn("no 'storage_config' entry on configuration .. will try an empty Map");
storageConfig = Maps.newHashMap();
}

storage = storageFactory.fromConfiguration(storageConfig);
} else if (DocumentStorage.class.isAssignableFrom(storageClass)) {
// if it was a storage ..
logger.info("got a DocumentStorage: " + storageClass + ". Trying default Constructor.");
storage = (DocumentStorage) storageClass.newInstance();
} else {
// if it was anything else .. complain
logger.error("got a storage option that is NOT a DocumentStorageFactory nor a DocumentStorage. I don't know what to do with it!");
throw new IllegalArgumentException("Configuration 'storage' class is not a DocumentStorageFactory nor a DocumentStorage.");
}
} catch (ClassNotFoundException e) {
throw new RuntimeException("DocumentStorage(Factory?) class not found", e);
} catch (SecurityException e) {
throw new RuntimeException("DocumentStorage(Factory?) class not instantiable", e);
} catch (InstantiationException e) {
throw new RuntimeException("DocumentStorage(Factory?) class threw an exception for the given configuration", e);
} catch (IllegalAccessException e) {
throw new RuntimeException("DocumentStorage(Factory?) class is not accessible", e);
}
}


return storage;
}


public BoostingIndexer getIndexer(){
return this.indexer;
Expand Down Expand Up @@ -486,14 +518,9 @@ private static Options getOptions(){

Option storage = OptionBuilder.withLongOpt("storage")
.hasArg()
.withDescription("if present, specifies a storage backend. Options are 'bdb' and 'ram'. Defaults to 'ram'.")
.withDescription("DEPRECATED! if present, specifies a storage backend. Only 'ram' is supported on command line. USE JSON CONFIGURATION!.")
.create("st");

Option bdbCache = OptionBuilder.withLongOpt("bdb-cache")
.hasArg()
.withDescription("if present, specifies the size of the berkeleyDb cache per thread, in megabytes. Defaults to 100MB.")
.create("bc");

Options options = new Options();
options.addOption(baseDir);
options.addOption(basePort);
Expand All @@ -512,7 +539,6 @@ private static Options getOptions(){
options.addOption(didyoumean);
options.addOption(configFile);
options.addOption(storage);
options.addOption(bdbCache);

return options;
}
Expand Down Expand Up @@ -563,6 +589,9 @@ public static void main(String[] args) throws IOException{
printHelp(getOptions(),null);
System.exit(1);
}


Map<Object, Object> configuration = Maps.newHashMap();

File baseDir = new File(line.getOptionValue("dir"));
int basePort = Integer.parseInt(line.getOptionValue("port", String.valueOf(DEFAULT_BASE_PORT)));
Expand All @@ -584,19 +613,20 @@ public static void main(String[] args) throws IOException{
suggest = SuggestValues.NO;
}

StorageValues storageValue = StorageValues.RAM;
int bdbCache = 0;
if (line.hasOption("storage")){
logger.warn("command-line option 'storage' is deprecated. write it on the JSON configuration!");
logger.warn("I'll try to do that for you this time .. ");

String storageType = line.getOptionValue("storage");
if ("bdb".equals(storageType)) {
storageValue = StorageValues.BDB;
bdbCache = Integer.parseInt(line.getOptionValue("bdb-cache", String.valueOf(DEFAULT_BDB_CACHE)));
} else if ("cassandra".equals(storageType)) {
storageValue = StorageValues.CASSANDRA;
} else if ("ram".equals(storageType)) {
storageValue = StorageValues.RAM;
if ("ram".equals(storageType)) {
Map<String, String> storageConfig = Maps.newHashMap();
storageConfig.put(InMemoryStorage.Factory.DIR, baseDir.getPath());
storageConfig.put(InMemoryStorage.Factory.LOAD, "true");

configuration.put("storage", InMemoryStorage.class.getName());
configuration.put("storage_config", storageConfig);
} else {
throw new IllegalArgumentException("storage has to be 'cassandra', 'bdb' or 'ram'. '" + storageType + "' given.");
throw new IllegalArgumentException("DEPRECATED command line storage got an Illegal value: " + storageType + ". Please migrate to JSON configuration!");
}
}

Expand All @@ -619,10 +649,9 @@ public static void main(String[] args) throws IOException{
String indexCode = line.getOptionValue("index-code");
logger.info("Command line option 'index-code' set to " + indexCode);

Map<Object, Object> configuration = Maps.newHashMap();

String configFile = line.getOptionValue("conf-file", null);
logger.info("Command line option 'conf-file' set to " + configFile);
logger.info("Command line option 'conf-file' set to " + configFile);

if (configFile != null) {
configuration = (Map<Object, Object>) JSONValue.parse(FileUtil.readFile(new File(configFile)));
Expand All @@ -634,8 +663,6 @@ public static void main(String[] args) throws IOException{
loadState,
boostsSize,
suggest,
storageValue,
bdbCache,
functions,
facets,
indexCode,
Expand Down
32 changes: 32 additions & 0 deletions src/main/java/com/flaptor/indextank/index/storage/InMemoryStorage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@

import com.flaptor.indextank.index.Document;
import com.flaptor.indextank.storage.alternatives.DocumentStorage;
import com.flaptor.indextank.storage.alternatives.DocumentStorageFactory;
import com.flaptor.util.Execute;
import com.flaptor.util.FileUtil;
import com.google.common.base.Preconditions;
Expand Down Expand Up @@ -192,5 +193,36 @@ public Map<String, String> getStats() {
stats.put("in_memory_storage_count", String.valueOf(compressedMap.size()));
return stats;
}


/**
* A {@link DocumnentStorageFactory} that returns {@link InMemoryStorage} classes.
*/
public static class Factory implements DocumentStorageFactory {

/** the KEY for backup directory on the config. it is <b>REQUIRED</b> */
public final static String DIR = "dir";
/** the KEY for 'load config from disk on startup' on the config. it is <b>REQUIRED</b> */
public final static String LOAD = "load";

@Override
public DocumentStorage fromConfiguration(Map<?, ?> config) {
Preconditions.checkNotNull(config);
Preconditions.checkState(!config.isEmpty(), "config cannot be empty");
Preconditions.checkNotNull(config.get(DIR), "config needs '" + DIR + "' entry");
Preconditions.checkNotNull(config.get(DIR), "config needs '" + LOAD + "' entry");


File backupDir = new File( config.get(DIR).toString() );
// toString is better than casting to String .. as the value _might_ be a boolean
Boolean load = Boolean.valueOf( config.get(LOAD).toString() );

try {
return new InMemoryStorage(backupDir, load);
} catch (IOException ioe) {
throw new RuntimeException("while creating InMemoryStorage: " + ioe.getMessage(), ioe);
}
}
}

}
7 changes: 7 additions & 0 deletions src/main/java/com/flaptor/indextank/storage/alternatives/DocumentStorageFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package com.flaptor.indextank.storage.alternatives;

import java.util.Map;

public interface DocumentStorageFactory {
public DocumentStorage fromConfiguration(Map<?, ?> config);
}
3 changes: 2 additions & 1 deletion src/test/java/com/flaptor/indextank/ResultsOrderingTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

import java.io.File;
import java.io.IOException;
import java.util.Map;
import java.util.Random;

import com.flaptor.indextank.index.Document;
Expand All @@ -44,7 +45,7 @@ public class ResultsOrderingTest extends IndexTankTestCase {
protected void setUp() throws Exception {
super.setUp();
this.tempDir = FileUtil.createTempDir("indextank","testcase");
this.indexEngine = new IndexEngine(this.tempDir, 11234, 100, false, 5, IndexEngine.SuggestValues.DOCUMENTS, IndexEngine.StorageValues.NO, 0, null, false, "dummyCode", "TEST-environment");
this.indexEngine = new IndexEngine(this.tempDir, 11234, 100, false, 5, IndexEngine.SuggestValues.DOCUMENTS, null, false, "dummyCode", "TEST-environment");
}

@Override
Expand Down
2 changes: 1 addition & 1 deletion src/test/java/com/flaptor/indextank/index/IndexEngineTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ public class IndexEngineTest extends IndexTankTestCase {
protected void setUp() throws Exception {
super.setUp();
this.tempDir = FileUtil.createTempDir("indextank","testcase");
this.indexEngine = new IndexEngine(this.tempDir, 11234, 5, false, 5, IndexEngine.SuggestValues.DOCUMENTS, IndexEngine.StorageValues.NO, 0, null, true, "dummyCode", "TEST-environment");
this.indexEngine = new IndexEngine(this.tempDir, 11234, 5, false, 5, IndexEngine.SuggestValues.DOCUMENTS, null, true, "dummyCode", "TEST-environment");
}

private void indexTwelveDocs(BoostingIndexer indexer) {
Expand Down
11 changes: 10 additions & 1 deletion src/test/java/com/flaptor/indextank/search/SnippetSearcherTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@

import java.io.File;
import java.io.IOException;
import java.util.Map;

import com.flaptor.indextank.DocumentStoringIndexer;
import com.flaptor.indextank.IndexTankTestCase;
Expand All @@ -47,7 +48,15 @@ public class SnippetSearcherTest extends IndexTankTestCase {
protected void setUp() throws Exception {
super.setUp();
this.tempDir = FileUtil.createTempDir("indextank","testcase");
this.indexEngine = new IndexEngine(this.tempDir, 11234, 5, false, 5, IndexEngine.SuggestValues.NO, IndexEngine.StorageValues.RAM, 0, null, false, "dummyCode", "TEST-environment");
Map<Object, Object> ieConfig = Maps.newHashMap();
Map<Object, Object> stConfig = Maps.newHashMap();
ieConfig.put("storage", com.flaptor.indextank.index.storage.InMemoryStorage.Factory.class.getName());
ieConfig.put("storage_config", stConfig);
stConfig.put(com.flaptor.indextank.index.storage.InMemoryStorage.Factory.DIR, this.tempDir.getPath());
stConfig.put(com.flaptor.indextank.index.storage.InMemoryStorage.Factory.LOAD, false);


this.indexEngine = new IndexEngine(this.tempDir, 11234, 5, false, 5, IndexEngine.SuggestValues.NO, null, false, "dummyCode", "TEST-environment", ieConfig);
this.indexer = new DocumentStoringIndexer(indexEngine.getIndexer(), indexEngine.getStorage());
this.searcher = new SnippetSearcher(indexEngine.getSearcher(), indexEngine.getStorage(), indexEngine.getParser());
}
Expand Down