Merge pull request #667 from cowchipkid/master

Incremental Training

curator/pom.xml

@@ -47,5 +47,4 @@
 
     </dependencies>
 
-
 </project>

dataless-classifier/pom.xml

@@ -67,5 +67,4 @@
         </pluginManagement>
     </build>
 
-
 </project>

ner/scripts/annotate.sh

@@ -54,7 +54,7 @@ configFile=$3
 # Classpath
 DIST=target
 LIB=target/dependency
-cpath=".:target/test-classes"
+cpath="./config/:.:target/test-classes"
 for JAR in `ls $DIST/*jar`; do
     cpath="$cpath:$JAR"
 done
@@ -66,4 +66,4 @@ CMD="java -classpath  ${cpath} -Xmx12g edu.illinois.cs.cogcomp.ner.NerTagger -an
 
 echo "$0: running command '$CMD'..."
 
-${CMD}
+${CMD}

ner/scripts/benchmarks.sh

@@ -30,7 +30,7 @@ if [ ! -e $LIB ]; then
 fi
 
 # Classpath
-cpath=".:target/classes"
+cpath="./config/:.:target/classes"
 
 for JAR in `ls $DIST/*.jar`; do
     cpath="$cpath:$JAR"
@@ -40,7 +40,7 @@ for JAR in `ls $LIB/*.jar`; do
     cpath="$cpath:$JAR"
 done
 
-CMD="java -classpath  ${cpath} -Xmx12g edu.illinois.cs.cogcomp.ner.NerBenchmark $1 $2 $3 $4 $5"
+CMD="java -classpath  ${cpath} -Xmx15g edu.illinois.cs.cogcomp.ner.NerBenchmark $1 $2 $3 $4 $5"
 
 echo "$0: running command '$CMD'..."
 

ner/scripts/runNER.sh

@@ -26,6 +26,7 @@ if [ ! -e $LIB ]; then
 fi
 
 # Build classpath
+cpath="./config/"
 for JAR in `ls $DIST/*jar`; do
     cpath="$cpath:$JAR"
 done
@@ -34,5 +35,6 @@ for JAR in `ls $LIB/*jar`; do
     cpath="$cpath:$JAR"
 done
 
-CMD="java -classpath  ${cpath} -Xms3g -Xmx10g -agentpath:/Applications/YourKit-Java-Profiler-2017.02.app/Contents/Resources/bin/mac/libyjpagent.jnilib edu.illinois.cs.cogcomp.ner.Main $1 $2"
+# YourKit profile -agentpath:/Applications/YourKit-Java-Profiler-2017.02.app/Contents/Resources/bin/mac/libyjpagent.jnilib
+CMD="java -classpath  ${cpath} -Xms3g -Xmx10g edu.illinois.cs.cogcomp.ner.Main $1 $2"
 $CMD

ner/scripts/test.sh

@@ -41,7 +41,7 @@ configFile=$2
 # Classpath
 DIST=target
 LIB=target/dependency
-cpath=".:target/test-classes"
+cpath="./config/:.:target/test-classes"
 for JAR in `ls $DIST/*jar`; do
     cpath="$cpath:$JAR"
 done

ner/scripts/train.sh

@@ -55,7 +55,7 @@ configFile=$3
 # Classpath
 DIST=target
 LIB=target/dependency
-cpath=".:target/test-classes"
+cpath="./config/:.:target/test-classes"
 for JAR in `ls $DIST/*jar`; do
     cpath="$cpath:$JAR"
 done

ner/src/main/java/edu/illinois/cs/cogcomp/ner/LbjTagger/LearningCurveMultiDataset.java

@@ -12,7 +12,6 @@
 import edu.illinois.cs.cogcomp.lbjava.learn.BatchTrainer;
 import edu.illinois.cs.cogcomp.lbjava.learn.SparseAveragedPerceptron;
 import edu.illinois.cs.cogcomp.lbjava.learn.SparseNetworkLearner;
-import edu.illinois.cs.cogcomp.lbjava.learn.featurepruning.SparseNetworkOptimizer;
 import edu.illinois.cs.cogcomp.lbjava.parse.Parser;
 import edu.illinois.cs.cogcomp.ner.ExpressiveFeatures.ExpressiveFeaturesAnnotator;
 import edu.illinois.cs.cogcomp.ner.ExpressiveFeatures.TwoLayerPredictionAggregationFeatures;
@@ -26,6 +25,11 @@
 import java.io.IOException;
 import java.util.Vector;
 
+/**
+ * build the model test it out, report results and so on.
+ * @author nick
+ * @author redman
+ */
 public class LearningCurveMultiDataset {
 
     private static final String NAME = LearningCurveMultiDataset.class.getCanonicalName();
@@ -42,7 +46,7 @@ public class LearningCurveMultiDataset {
      * @param devDataPath data used to auto-converge.
      */
     public static void buildFinalModel(int fixedNumIterations, String trainDataPath,
-            String testDataPath, String devDataPath) throws Exception {
+            String testDataPath, String devDataPath, boolean incremental) throws Exception {
         Data trainData = new Data(trainDataPath, trainDataPath, "-c", new String[] {}, new String[] {});
         ExpressiveFeaturesAnnotator.annotate(trainData);
         Data testData = new Data(testDataPath, testDataPath, "-c", new String[] {}, new String[] {});
@@ -56,7 +60,7 @@ public static void buildFinalModel(int fixedNumIterations, String trainDataPath,
         test.addElement(devData);
         logger.debug("Building final model: iterations = " + fixedNumIterations + " train = '"
                         + trainDataPath + "' test = '"+testDataPath+"' dev = '" + testDataPath+"'");
-        getLearningCurve(train, test, fixedNumIterations);
+        getLearningCurve(train, test, fixedNumIterations, incremental);
     }
 
     /**
@@ -65,7 +69,7 @@ public static void buildFinalModel(int fixedNumIterations, String trainDataPath,
      * use fixedNumIterations=-1 if you want to use the automatic convergence criterion
      */
     public static void getLearningCurve(int fixedNumIterations, String trainDataPath,
-            String testDataPath) throws Exception {
+            String testDataPath, boolean incremental) throws Exception {
         logger.debug("getLearningCurve(): fni = " + fixedNumIterations + " trainDataPath = '"
                 + trainDataPath + "' testDataPath = '" + testDataPath + "'....");
         Data trainData =
@@ -78,16 +82,18 @@ public static void getLearningCurve(int fixedNumIterations, String trainDataPath
         train.addElement(trainData);
         Vector<Data> test = new Vector<>();
         test.addElement(testData);
-        getLearningCurve(train, test, fixedNumIterations);
+        getLearningCurve(train, test, fixedNumIterations, incremental);
     }
 
     /**
-     * use fixedNumIterations=-1 if you want to use the automatic convergence criterion
+     * use fixedNumIterations=-1 if you want to use the automatic convergence criterion, incremental
+     * true will start with the existing models weights, and continue training with that set of default
+     * weights.
      * <p>
      * NB: assuming column format
      */
     public static void getLearningCurve(Vector<Data> trainDataSet, Vector<Data> testDataSet,
-            int fixedNumIterations) throws Exception {
+            int fixedNumIterations, boolean incremental) throws Exception {
         double bestF1Level1 = -1;
         int bestRoundLevel1 = 0;
         // Get the directory name (<configname>.model is appended in LbjTagger/Parameters.java:139)
@@ -114,9 +120,15 @@ public static void getLearningCurve(Vector<Data> trainDataSet, Vector<Data> test
         paramLevel1.baseLTU.featurePruningThreshold = ParametersForLbjCode.currentParameters.featurePruningThreshold;
         logger.info("Level 1 classifier learning rate = "+ParametersForLbjCode.currentParameters.learningRatePredictionsLevel1+
             ", thickness = "+ParametersForLbjCode.currentParameters.thicknessPredictionsLevel1);
+
         NETaggerLevel1 tagger1 =
                 new NETaggerLevel1(paramLevel1, modelPath + ".level1", modelPath + ".level1.lex");
-        tagger1.forget();
+        if (!incremental) {
+            logger.info("Training L1 model from scratch.");
+            tagger1.forget();
+        } else {
+            logger.info("Training L1 model incrementally.");
+        }
         ParametersForLbjCode.currentParameters.taggerLevel1 = tagger1;
         for (int dataId = 0; dataId < trainDataSet.size(); dataId++) {
             Data trainData = trainDataSet.elementAt(dataId);
@@ -192,7 +204,12 @@ public static void getLearningCurve(Vector<Data> trainDataSet, Vector<Data> test
                 new NETaggerLevel2(paramLevel2, ParametersForLbjCode.currentParameters.pathToModelFile
                         + ".level2", ParametersForLbjCode.currentParameters.pathToModelFile
                         + ".level2.lex");
-        tagger2.forget();
+        if (!incremental) {
+            logger.info("Training L2 model from scratch.");
+            tagger2.forget();
+        } else {
+            logger.info("Training L2 model incrementally.");
+        }
         ParametersForLbjCode.currentParameters.taggerLevel2 = tagger2;
 
         // Previously checked if PatternFeatures was in featuresToUse.

ner/src/main/java/edu/illinois/cs/cogcomp/ner/LbjTagger/NETesterMultiDataset.java

@@ -45,12 +45,6 @@ public static void test(String testDatapath, boolean verbose,
             data.elementAt(0).setLabelsToAnonymize(labelsToAnonymizeInEvaluation);
         NETaggerLevel1 taggerLevel1 = (NETaggerLevel1) ParametersForLbjCode.currentParameters.taggerLevel1;
         NETaggerLevel2 taggerLevel2 = (NETaggerLevel2) ParametersForLbjCode.currentParameters.taggerLevel2;
-        SparseAveragedPerceptron sap1 = (SparseAveragedPerceptron)taggerLevel1.getBaseLTU();
-        System.out.println("L1 SparseAveragedPerceptron learning rate = "+sap1.getLearningRate()+", thickness = "+sap1.getPositiveThickness());
-        if (ParametersForLbjCode.currentParameters.featuresToUse.containsKey("PredictionsLevel1")) {
-            SparseAveragedPerceptron sap2 = (SparseAveragedPerceptron)taggerLevel2.getBaseLTU();
-            System.out.println("L2 SparseAveragedPerceptron learning rate = "+sap2.getLearningRate()+", thickness = "+sap2.getPositiveThickness());
-        }
         printTestResultsByDataset(data, taggerLevel1, taggerLevel2, verbose);
     }
 

ner/src/main/java/edu/illinois/cs/cogcomp/ner/ModelLoader.java

@@ -38,8 +38,9 @@ public class ModelLoader {
      * from Minio datastore.
      * @param rm the resource manager.
      * @param viewName the name of the view identifies the model.
+     * @param training if true, we are going to train the model, if it doesn't exist, create it.
      */
-    static public void load(ResourceManager rm, String viewName) {
+    static public void load(ResourceManager rm, String viewName, boolean training) {
 
         // the loaded built into the model will check the local file system and the jar files in the classpath.
         ParametersForLbjCode cp = ParametersForLbjCode.currentParameters;
@@ -65,9 +66,21 @@ static public void load(ResourceManager rm, String viewName) {
             } else {
                 logger.info("L2 model not required.");
             }
+        } else if (training) {
+
+            // we are training a new model, so it it doesn't exist, we don't care, just create a
+            // container.
+            tagger1 = new NETaggerLevel1(modelPath + ".level1", modelPath + ".level1.lex");
+            logger.info("Reading L1 model from file : " + modelPath + ".level2");
+            if (cp.featuresToUse.containsKey("PredictionsLevel1")) {
+                tagger2 = new NETaggerLevel2(modelPath + ".level2", modelPath + ".level2.lex");
+                logger.info("Reading L2 model from file : " + modelPath + ".level2");
+            } else {
+                logger.info("L2 model not required.");
+            }
         } else {
 
-            // all else has filed, load from the datastore, create artifact ids based on the view
+            // all else has failed, load from the datastore, create artifact ids based on the view
             // name and training data designation.
             String dataset;
             String lowercaseViewName = viewName.toLowerCase();

ner/src/main/java/edu/illinois/cs/cogcomp/ner/NERAnnotator.java

@@ -102,7 +102,7 @@ public void initialize(ResourceManager nerRm) {
         Parameters.readConfigAndLoadExternalData(nerRm);
 
         // load the models.
-        ModelLoader.load(nerRm, viewName);
+        ModelLoader.load(nerRm, viewName, false);
     }
 
     /**

ner/src/main/java/edu/illinois/cs/cogcomp/ner/NerBenchmark.java

@@ -8,6 +8,9 @@
 package edu.illinois.cs.cogcomp.ner;
 
 import edu.illinois.cs.cogcomp.core.utilities.configuration.ResourceManager;
+import edu.illinois.cs.cogcomp.lbjava.learn.SparseAveragedPerceptron;
+import edu.illinois.cs.cogcomp.ner.LbjFeatures.NETaggerLevel1;
+import edu.illinois.cs.cogcomp.ner.LbjFeatures.NETaggerLevel2;
 import edu.illinois.cs.cogcomp.ner.LbjTagger.*;
 import edu.illinois.cs.cogcomp.ner.config.NerBaseConfigurator;
 
@@ -44,6 +47,7 @@
  * -features : for debugging, reports the feature vector for each token in the dataset. Output produced in a "features.out" file.
  * -iterations : specify a fixed number of iterations, or -1 (the default) means auto converge requiring a "dev" directory.
  * -release : build a final model for release, it will build on test and train, and unless "-iterations" specified, it will autoconvert
+ * -incremental : rather than discarding existing weights, start with those that already exist and continue training.
  * using "dev" for a holdout set.
  * }
  */
@@ -70,6 +74,10 @@ public class NerBenchmark {
     /** Report the input features for each level */
     static boolean verbose = false;
 
+    /** If this is set, we will start with the existing weights (and averages) for the 
+     * model and continue training from there. */
+    static boolean incremental = false;
+
     /** the output file name. */
     static String output = null;
 
@@ -94,6 +102,10 @@ private static void parseArguments(String[] args) {
                     }
                     directory = args[i];
                     break;
+                case "-incremental":
+                    System.out.println("Configured for incremental training.");
+                    incremental = true;
+                    break;
                 case "-verbose":
                     verbose = true;
                     break;
@@ -184,11 +196,25 @@ public boolean accept(File dir, String name) {
                             System.out.println("\n\n----- Training models for evaluation for "+confFile+" ------");
                             Parameters.readConfigAndLoadExternalData(confFile, !skiptraining);
                             ResourceManager rm = new ResourceManager(confFile);
-                            ModelLoader.load(rm, rm.getString("modelName"));
+                            ModelLoader.load(rm, rm.getString("modelName"), !skiptraining);
 
+                            // report learning rates and thicknesses
+                            NETaggerLevel1 taggerLevel1 = (NETaggerLevel1) ParametersForLbjCode.currentParameters.taggerLevel1;
+                            NETaggerLevel2 taggerLevel2 = (NETaggerLevel2) ParametersForLbjCode.currentParameters.taggerLevel2;
+                            SparseAveragedPerceptron sap1 = (SparseAveragedPerceptron)taggerLevel1.getBaseLTU();
+                            sap1.setLearningRate(ParametersForLbjCode.currentParameters.learningRatePredictionsLevel1);
+                            sap1.setThickness(ParametersForLbjCode.currentParameters.thicknessPredictionsLevel1);
+                            System.out.println("L1 learning rate = "+sap1.getLearningRate()+", thickness = "+sap1.getPositiveThickness());
+                            if (ParametersForLbjCode.currentParameters.featuresToUse.containsKey("PredictionsLevel1")) {
+                                SparseAveragedPerceptron sap2 = (SparseAveragedPerceptron)taggerLevel2.getBaseLTU();
+                                sap2.setLearningRate(ParametersForLbjCode.currentParameters.learningRatePredictionsLevel2);
+                                sap2.setThickness(ParametersForLbjCode.currentParameters.thicknessPredictionsLevel2);
+                                System.out.println("L2 learning rate = "+sap2.getLearningRate()+", thickness = "+sap2.getPositiveThickness());
+                            }
+
                             // there is a training directory, with training enabled, so train. We use the same dataset
                             // for both training and evaluating.
-                            LearningCurveMultiDataset.getLearningCurve(iterations, trainDirName, devDirName);
+                            LearningCurveMultiDataset.getLearningCurve(iterations, trainDirName, devDirName, incremental);
                             System.out.println("\n\n----- Final results for "+confFile+", verbose ------");
                             NETesterMultiDataset.test(testDirName, true,
                                     ParametersForLbjCode.currentParameters.labelsToIgnoreInEvaluation,
@@ -205,7 +231,7 @@ public boolean accept(File dir, String name) {
                         System.out.println("\n\n----- Reporting results from existing models for "+confFile+" ------");
                         Parameters.readConfigAndLoadExternalData(confFile, !skiptraining);
                         ResourceManager rm = new ResourceManager(confFile);
-                        ModelLoader.load(rm, rm.getString("modelName"));
+                        ModelLoader.load(rm, rm.getString("modelName"), !skiptraining);
                         System.out.println("Benchmark against configuration : " + confFile);
                         if (reportLabels)
                             NEDisplayPredictions.test(testDirName, "-c", verbose);
@@ -221,12 +247,12 @@ else if (reportFeatures)
                         if (trainDir.exists() && testDir.exists() && devDir.exists()) {
                             Parameters.readConfigAndLoadExternalData(confFile, !skiptraining);
                             ResourceManager rm = new ResourceManager(confFile);
-                            ModelLoader.load(rm, rm.getString("modelName"));
+                            ModelLoader.load(rm, rm.getString("modelName"), true);
                             System.out.println("\n\n----- Building a final model for "+confFile+" ------");
 
                             // there is a training directory, with training enabled, so train. We use the same dataset
                             // for both training and evaluating.
-                            LearningCurveMultiDataset.buildFinalModel(iterations, trainDirName, testDirName, devDirName);
+                            LearningCurveMultiDataset.buildFinalModel(iterations, trainDirName, testDirName, devDirName, incremental);
                             System.out.println("\n\n----- Release results for "+confFile+", verbose ------");
                             NETesterMultiDataset.test(devDirName, true,
                                     ParametersForLbjCode.currentParameters.labelsToIgnoreInEvaluation,

ner/src/main/java/edu/illinois/cs/cogcomp/ner/NerTagger.java

@@ -38,13 +38,13 @@ public static void main(String[] args) {
             Parameters.readConfigAndLoadExternalData(args[args.length - 1], areWeTraining);
 
             if (args[0].equalsIgnoreCase("-train"))
-                LearningCurveMultiDataset.getLearningCurve(-1, args[1], args[2]);
+                LearningCurveMultiDataset.getLearningCurve(-1, args[1], args[2], false);
             else if (args[0].equalsIgnoreCase("-trainFixedIterations"))
                 LearningCurveMultiDataset.getLearningCurve(Integer.parseInt(args[1]), args[2],
-                        args[3]);
+                        args[3], false);
             else {
                 // load up the models
-                ModelLoader.load(rm, rm.getString("modelName"));
+                ModelLoader.load(rm, rm.getString("modelName"), false);
                 if (args[0].equalsIgnoreCase("-annotate")) {
                     NETagPlain.init();
                     NETagPlain.tagData(args[1], args[2]);

ner/src/test/java/edu/illinois/cs/cogcomp/ner/NerTest.java

@@ -44,7 +44,7 @@ public void setUp() throws Exception {
             ResourceManager rm = new NerBaseConfigurator().getDefaultConfig();
             Parameters.readConfigAndLoadExternalData(rm);
             ParametersForLbjCode.currentParameters.forceNewSentenceOnLineBreaks = false;
-            ModelLoader.load(rm, ViewNames.NER_ONTONOTES);
+            ModelLoader.load(rm, ViewNames.NER_ONTONOTES, false);
             t1 = (NETaggerLevel1) ParametersForLbjCode.currentParameters.taggerLevel1;
             t2 = (NETaggerLevel2) ParametersForLbjCode.currentParameters.taggerLevel2;
         } catch (Exception e) {

similarity/pom.xml

@@ -68,6 +68,4 @@
             <version>1.0</version>
         </dependency>
     </dependencies>
-
-
 </project>

verbsense/pom.xml

@@ -145,5 +145,4 @@
             </plugins>
         </pluginManagement>
     </build>
-
 </project>