From 2912e63192dc30fbae90a41087c74a6b422f0ac2 Mon Sep 17 00:00:00 2001 From: Diego Buthay Date: Tue, 14 Feb 2012 15:26:04 -0800 Subject: [PATCH 1/3] making storage engines pluggable --- .../flaptor/indextank/index/IndexEngine.java | 107 +++++++++++------- .../index/storage/InMemoryStorage.java | 32 ++++++ .../alternatives/DocumentStorageFactory.java | 7 ++ 3 files changed, 106 insertions(+), 40 deletions(-) create mode 100644 src/main/java/com/flaptor/indextank/storage/alternatives/DocumentStorageFactory.java diff --git a/src/main/java/com/flaptor/indextank/index/IndexEngine.java b/src/main/java/com/flaptor/indextank/index/IndexEngine.java index 22358f7..1cdae30 100644 --- a/src/main/java/com/flaptor/indextank/index/IndexEngine.java +++ b/src/main/java/com/flaptor/indextank/index/IndexEngine.java @@ -63,6 +63,7 @@ import com.flaptor.indextank.search.SnippetSearcher; import com.flaptor.indextank.search.TrafficLimitingSearcher; import com.flaptor.indextank.storage.alternatives.DocumentStorage; +import com.flaptor.indextank.storage.alternatives.DocumentStorageFactory; import com.flaptor.indextank.suggest.DidYouMeanSuggestor; import com.flaptor.indextank.suggest.NoSuggestor; import com.flaptor.indextank.suggest.QuerySuggestor; @@ -112,10 +113,8 @@ public class IndexEngine { private static final int DEFAULT_BASE_PORT = 7910; private static final int DEFAULT_RTI_SIZE = 1000; - private static final int DEFAULT_BDB_CACHE = 100; public static enum SuggestValues { NO, QUERIES, DOCUMENTS}; - public static enum StorageValues { NO, BDB, RAM, CASSANDRA }; public IndexEngine( File baseDir, int basePort, @@ -123,8 +122,6 @@ public IndexEngine( File baseDir, boolean load, int boostsSize, SuggestValues suggest, - StorageValues storageValue, - int bdbCache, String functions, boolean facets, String indexCode, @@ -136,8 +133,6 @@ public IndexEngine( File baseDir, load, boostsSize, suggest, - storageValue, - bdbCache, functions, facets, indexCode, @@ -152,8 +147,6 @@ public IndexEngine( File baseDir, boolean load, int boostsSize, SuggestValues suggest, - StorageValues storageValue, - int bdbCache, String functions, boolean facets, String indexCode, @@ -276,18 +269,9 @@ public IndexEngine( File baseDir, logger.info("Index recovery configuration set to recover index from simpleDB"); this.recoveryStorage = IndexRecoverer.IndexStorageValue.SIMPLEDB; } - - switch (storageValue) { - case RAM: - storage = new InMemoryStorage(baseDir, load); - logger.info("Using in-memory storage"); - break; - case NO: - storage = null; - logger.info("NOT Using storage"); - break; - } - + + storage = buildStorage(configuration); + promoter = new BasicPromoter(baseDir, load); searcher = new Blender(lsi, rti, suggestor, promoter, boostsManager); indexer = new Dealer(lsi, rti, suggestor, boostsManager, rtiSize, promoter, functionsManager); @@ -326,6 +310,54 @@ private Analyzer buildAnalyzer(Map configuration) { return analyzer; }; + public DocumentStorage buildStorage(Map configuration) { + + DocumentStorage storage = null; + + // parse storage from configuration + String storageClassStr = (String) configuration.get("storage"); + if (null == storageClassStr || "".equals(storageClassStr.trim()) ) { + logger.info("NOT Using storage"); + } else { + try { + // it is either a factory, or a concrete class + Class storageClass = Class.forName(storageClassStr); + + if (DocumentStorageFactory.class.isAssignableFrom(storageClass)) { + // if it was a factory .. + logger.info("got a DocumentStorageFactory: " + storageClass); + DocumentStorageFactory storageFactory = (DocumentStorageFactory) storageClass.newInstance(); + Map storageConfig = (Map)configuration.get("storage_config"); + if (null == storageConfig) { + logger.warn("no 'storage_config' entry on configuration .. will try an empty Map"); + storageConfig = Maps.newHashMap(); + } + + storage = storageFactory.fromConfiguration(storageConfig); + } else if (DocumentStorage.class.isAssignableFrom(storageClass)) { + // if it was a storage .. + logger.info("got a DocumentStorage: " + storageClass + ". Trying default Constructor."); + storage = (DocumentStorage) storageClass.newInstance(); + } else { + // if it was anything else .. complain + logger.error("got a storage option that is NOT a DocumentStorageFactory nor a DocumentStorage. I don't know what to do with it!"); + throw new IllegalArgumentException("Configuration 'storage' class is not a DocumentStorageFactory nor a DocumentStorage."); + } + } catch (ClassNotFoundException e) { + throw new RuntimeException("DocumentStorage(Factory?) class not found", e); + } catch (SecurityException e) { + throw new RuntimeException("DocumentStorage(Factory?) class not instantiable", e); + } catch (InstantiationException e) { + throw new RuntimeException("DocumentStorage(Factory?) class threw an exception for the given configuration", e); + } catch (IllegalAccessException e) { + throw new RuntimeException("DocumentStorage(Factory?) class is not accessible", e); + } + } + + + return storage; + } + public BoostingIndexer getIndexer(){ return this.indexer; @@ -489,11 +521,6 @@ private static Options getOptions(){ .withDescription("if present, specifies a storage backend. Options are 'bdb' and 'ram'. Defaults to 'ram'.") .create("st"); - Option bdbCache = OptionBuilder.withLongOpt("bdb-cache") - .hasArg() - .withDescription("if present, specifies the size of the berkeleyDb cache per thread, in megabytes. Defaults to 100MB.") - .create("bc"); - Options options = new Options(); options.addOption(baseDir); options.addOption(basePort); @@ -512,7 +539,6 @@ private static Options getOptions(){ options.addOption(didyoumean); options.addOption(configFile); options.addOption(storage); - options.addOption(bdbCache); return options; } @@ -563,6 +589,9 @@ public static void main(String[] args) throws IOException{ printHelp(getOptions(),null); System.exit(1); } + + + Map configuration = Maps.newHashMap(); File baseDir = new File(line.getOptionValue("dir")); int basePort = Integer.parseInt(line.getOptionValue("port", String.valueOf(DEFAULT_BASE_PORT))); @@ -584,19 +613,20 @@ public static void main(String[] args) throws IOException{ suggest = SuggestValues.NO; } - StorageValues storageValue = StorageValues.RAM; - int bdbCache = 0; if (line.hasOption("storage")){ + logger.warn("command-line option 'storage' is deprecated. write it on the JSON configuration!"); + logger.warn("I'll try to do that for you this time .. "); + String storageType = line.getOptionValue("storage"); - if ("bdb".equals(storageType)) { - storageValue = StorageValues.BDB; - bdbCache = Integer.parseInt(line.getOptionValue("bdb-cache", String.valueOf(DEFAULT_BDB_CACHE))); - } else if ("cassandra".equals(storageType)) { - storageValue = StorageValues.CASSANDRA; - } else if ("ram".equals(storageType)) { - storageValue = StorageValues.RAM; + if ("ram".equals(storageType)) { + Map storageConfig = Maps.newHashMap(); + storageConfig.put(InMemoryStorage.Factory.DIR, baseDir.getPath()); + storageConfig.put(InMemoryStorage.Factory.LOAD, "true"); + + configuration.put("storage", InMemoryStorage.class.getName()); + configuration.put("storage_config", storageConfig); } else { - throw new IllegalArgumentException("storage has to be 'cassandra', 'bdb' or 'ram'. '" + storageType + "' given."); + throw new IllegalArgumentException("DEPRECATED command line storage got an Illegal value: " + storageType + ". Please migrate to JSON configuration!"); } } @@ -619,10 +649,9 @@ public static void main(String[] args) throws IOException{ String indexCode = line.getOptionValue("index-code"); logger.info("Command line option 'index-code' set to " + indexCode); - Map configuration = Maps.newHashMap(); String configFile = line.getOptionValue("conf-file", null); - logger.info("Command line option 'conf-file' set to " + configFile); + logger.info("Command line option 'conf-file' set to " + configFile); if (configFile != null) { configuration = (Map) JSONValue.parse(FileUtil.readFile(new File(configFile))); @@ -634,8 +663,6 @@ public static void main(String[] args) throws IOException{ loadState, boostsSize, suggest, - storageValue, - bdbCache, functions, facets, indexCode, diff --git a/src/main/java/com/flaptor/indextank/index/storage/InMemoryStorage.java b/src/main/java/com/flaptor/indextank/index/storage/InMemoryStorage.java index 89eb6cc..612ac9c 100644 --- a/src/main/java/com/flaptor/indextank/index/storage/InMemoryStorage.java +++ b/src/main/java/com/flaptor/indextank/index/storage/InMemoryStorage.java @@ -42,6 +42,7 @@ import com.flaptor.indextank.index.Document; import com.flaptor.indextank.storage.alternatives.DocumentStorage; +import com.flaptor.indextank.storage.alternatives.DocumentStorageFactory; import com.flaptor.util.Execute; import com.flaptor.util.FileUtil; import com.google.common.base.Preconditions; @@ -192,5 +193,36 @@ public Map getStats() { stats.put("in_memory_storage_count", String.valueOf(compressedMap.size())); return stats; } + + + /** + * A {@link DocumnentStorageFactory} that returns {@link InMemoryStorage} classes. + */ + public class Factory implements DocumentStorageFactory { + + /** the KEY for backup directory on the config. it is REQUIRED */ + public final static String DIR = "dir"; + /** the KEY for 'load config from disk on startup' on the config. it is REQUIRED */ + public final static String LOAD = "load"; + + @Override + public DocumentStorage fromConfiguration(Map config) { + Preconditions.checkNotNull(config); + Preconditions.checkState(!config.isEmpty(), "config cannot be empty"); + Preconditions.checkNotNull(config.get(DIR), "config needs '" + DIR + "' entry"); + Preconditions.checkNotNull(config.get(DIR), "config needs '" + LOAD + "' entry"); + + + File backupDir = new File( config.get(DIR).toString() ); + // toString is better than casting to String .. as the value _might_ be a boolean + Boolean load = Boolean.valueOf( config.get(LOAD).toString() ); + + try { + return new InMemoryStorage(backupDir, load); + } catch (IOException ioe) { + throw new RuntimeException("while creating InMemoryStorage: " + ioe.getMessage(), ioe); + } + } + } } diff --git a/src/main/java/com/flaptor/indextank/storage/alternatives/DocumentStorageFactory.java b/src/main/java/com/flaptor/indextank/storage/alternatives/DocumentStorageFactory.java new file mode 100644 index 0000000..08e4777 --- /dev/null +++ b/src/main/java/com/flaptor/indextank/storage/alternatives/DocumentStorageFactory.java @@ -0,0 +1,7 @@ +package com.flaptor.indextank.storage.alternatives; + +import java.util.Map; + +public interface DocumentStorageFactory { + public DocumentStorage fromConfiguration(Map config); +} From 117bd47571f2d13d2a9e04b42c2f317a1968f9e3 Mon Sep 17 00:00:00 2001 From: Diego Buthay Date: Tue, 14 Feb 2012 15:32:19 -0800 Subject: [PATCH 2/3] changing help for --storage cli argument --- src/main/java/com/flaptor/indextank/index/IndexEngine.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/flaptor/indextank/index/IndexEngine.java b/src/main/java/com/flaptor/indextank/index/IndexEngine.java index 1cdae30..0cf8fbe 100644 --- a/src/main/java/com/flaptor/indextank/index/IndexEngine.java +++ b/src/main/java/com/flaptor/indextank/index/IndexEngine.java @@ -518,7 +518,7 @@ private static Options getOptions(){ Option storage = OptionBuilder.withLongOpt("storage") .hasArg() - .withDescription("if present, specifies a storage backend. Options are 'bdb' and 'ram'. Defaults to 'ram'.") + .withDescription("DEPRECATED! if present, specifies a storage backend. Only 'ram' is supported on command line. USE JSON CONFIGURATION!.") .create("st"); Options options = new Options(); From 25908d8fd0b92e3c9493b48c059d65f08094f26c Mon Sep 17 00:00:00 2001 From: Diego Buthay Date: Wed, 15 Feb 2012 14:56:29 -0800 Subject: [PATCH 3/3] fixing unit tests .. they all pass now --- .../indextank/index/storage/InMemoryStorage.java | 2 +- .../com/flaptor/indextank/ResultsOrderingTest.java | 3 ++- .../com/flaptor/indextank/index/IndexEngineTest.java | 2 +- .../flaptor/indextank/search/SnippetSearcherTest.java | 11 ++++++++++- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/flaptor/indextank/index/storage/InMemoryStorage.java b/src/main/java/com/flaptor/indextank/index/storage/InMemoryStorage.java index 612ac9c..ba389b2 100644 --- a/src/main/java/com/flaptor/indextank/index/storage/InMemoryStorage.java +++ b/src/main/java/com/flaptor/indextank/index/storage/InMemoryStorage.java @@ -198,7 +198,7 @@ public Map getStats() { /** * A {@link DocumnentStorageFactory} that returns {@link InMemoryStorage} classes. */ - public class Factory implements DocumentStorageFactory { + public static class Factory implements DocumentStorageFactory { /** the KEY for backup directory on the config. it is REQUIRED */ public final static String DIR = "dir"; diff --git a/src/test/java/com/flaptor/indextank/ResultsOrderingTest.java b/src/test/java/com/flaptor/indextank/ResultsOrderingTest.java index c4c86f9..de65e60 100644 --- a/src/test/java/com/flaptor/indextank/ResultsOrderingTest.java +++ b/src/test/java/com/flaptor/indextank/ResultsOrderingTest.java @@ -21,6 +21,7 @@ import java.io.File; import java.io.IOException; +import java.util.Map; import java.util.Random; import com.flaptor.indextank.index.Document; @@ -44,7 +45,7 @@ public class ResultsOrderingTest extends IndexTankTestCase { protected void setUp() throws Exception { super.setUp(); this.tempDir = FileUtil.createTempDir("indextank","testcase"); - this.indexEngine = new IndexEngine(this.tempDir, 11234, 100, false, 5, IndexEngine.SuggestValues.DOCUMENTS, IndexEngine.StorageValues.NO, 0, null, false, "dummyCode", "TEST-environment"); + this.indexEngine = new IndexEngine(this.tempDir, 11234, 100, false, 5, IndexEngine.SuggestValues.DOCUMENTS, null, false, "dummyCode", "TEST-environment"); } @Override diff --git a/src/test/java/com/flaptor/indextank/index/IndexEngineTest.java b/src/test/java/com/flaptor/indextank/index/IndexEngineTest.java index 5618256..a76dd43 100644 --- a/src/test/java/com/flaptor/indextank/index/IndexEngineTest.java +++ b/src/test/java/com/flaptor/indextank/index/IndexEngineTest.java @@ -57,7 +57,7 @@ public class IndexEngineTest extends IndexTankTestCase { protected void setUp() throws Exception { super.setUp(); this.tempDir = FileUtil.createTempDir("indextank","testcase"); - this.indexEngine = new IndexEngine(this.tempDir, 11234, 5, false, 5, IndexEngine.SuggestValues.DOCUMENTS, IndexEngine.StorageValues.NO, 0, null, true, "dummyCode", "TEST-environment"); + this.indexEngine = new IndexEngine(this.tempDir, 11234, 5, false, 5, IndexEngine.SuggestValues.DOCUMENTS, null, true, "dummyCode", "TEST-environment"); } private void indexTwelveDocs(BoostingIndexer indexer) { diff --git a/src/test/java/com/flaptor/indextank/search/SnippetSearcherTest.java b/src/test/java/com/flaptor/indextank/search/SnippetSearcherTest.java index 490445a..6ccd9c4 100644 --- a/src/test/java/com/flaptor/indextank/search/SnippetSearcherTest.java +++ b/src/test/java/com/flaptor/indextank/search/SnippetSearcherTest.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.IOException; +import java.util.Map; import com.flaptor.indextank.DocumentStoringIndexer; import com.flaptor.indextank.IndexTankTestCase; @@ -47,7 +48,15 @@ public class SnippetSearcherTest extends IndexTankTestCase { protected void setUp() throws Exception { super.setUp(); this.tempDir = FileUtil.createTempDir("indextank","testcase"); - this.indexEngine = new IndexEngine(this.tempDir, 11234, 5, false, 5, IndexEngine.SuggestValues.NO, IndexEngine.StorageValues.RAM, 0, null, false, "dummyCode", "TEST-environment"); + Map ieConfig = Maps.newHashMap(); + Map stConfig = Maps.newHashMap(); + ieConfig.put("storage", com.flaptor.indextank.index.storage.InMemoryStorage.Factory.class.getName()); + ieConfig.put("storage_config", stConfig); + stConfig.put(com.flaptor.indextank.index.storage.InMemoryStorage.Factory.DIR, this.tempDir.getPath()); + stConfig.put(com.flaptor.indextank.index.storage.InMemoryStorage.Factory.LOAD, false); + + + this.indexEngine = new IndexEngine(this.tempDir, 11234, 5, false, 5, IndexEngine.SuggestValues.NO, null, false, "dummyCode", "TEST-environment", ieConfig); this.indexer = new DocumentStoringIndexer(indexEngine.getIndexer(), indexEngine.getStorage()); this.searcher = new SnippetSearcher(indexEngine.getSearcher(), indexEngine.getStorage(), indexEngine.getParser()); }