diff --git a/big-data-utils/pom.xml b/big-data-utils/pom.xml index cb0afdd80..7387d132d 100644 --- a/big-data-utils/pom.xml +++ b/big-data-utils/pom.xml @@ -3,7 +3,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 4.0.0 @@ -23,7 +23,7 @@ edu.illinois.cs.cogcomp illinois-core-utilities - 4.0.11 + 4.0.12 org.xeustechnologies.google-api diff --git a/chunker/pom.xml b/chunker/pom.xml index f254f67ba..dca212c09 100644 --- a/chunker/pom.xml +++ b/chunker/pom.xml @@ -2,7 +2,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 4.0.0 @@ -13,7 +13,7 @@ edu.illinois.cs.cogcomp illinois-core-utilities - 4.0.11 + 4.0.12 @@ -24,12 +24,12 @@ edu.illinois.cs.cogcomp LBJava-NLP-tools - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-pos - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp diff --git a/commasrl/pom.xml b/commasrl/pom.xml index c587699c0..33b939ff0 100644 --- a/commasrl/pom.xml +++ b/commasrl/pom.xml @@ -4,7 +4,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 4.0.0 @@ -35,48 +35,48 @@ edu.illinois.cs.cogcomp illinois-core-utilities - 4.0.11 + 4.0.12 true edu.illinois.cs.cogcomp illinois-curator - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-tokenizer - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-corpusreaders - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-inference - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp stanford_3.3.1 - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-pos - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-ner - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-chunker - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp diff --git a/core-utilities/pom.xml b/core-utilities/pom.xml index 959ac6190..a0a87217e 100644 --- a/core-utilities/pom.xml +++ b/core-utilities/pom.xml @@ -6,7 +6,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 illinois-core-utilities diff --git a/corpusreaders/pom.xml b/corpusreaders/pom.xml index fbaf1eed2..cc04aabf1 100644 --- a/corpusreaders/pom.xml +++ b/corpusreaders/pom.xml @@ -6,7 +6,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 illinois-corpusreaders @@ -15,12 +15,12 @@ edu.illinois.cs.cogcomp illinois-core-utilities - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-tokenizer - 4.0.11 + 4.0.12 org.slf4j diff --git a/curator/pom.xml b/curator/pom.xml index fda6fea20..9e7f6b8c3 100644 --- a/curator/pom.xml +++ b/curator/pom.xml @@ -7,7 +7,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 illinois-curator @@ -16,7 +16,7 @@ edu.illinois.cs.cogcomp illinois-core-utilities - 4.0.11 + 4.0.12 diff --git a/dataless-classifier/pom.xml b/dataless-classifier/pom.xml index 66e3f9d0f..1600e01ab 100644 --- a/dataless-classifier/pom.xml +++ b/dataless-classifier/pom.xml @@ -3,7 +3,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 4.0.0 @@ -21,12 +21,12 @@ edu.illinois.cs.cogcomp illinois-core-utilities - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-tokenizer - 4.0.11 + 4.0.12 org.slf4j diff --git a/depparse/pom.xml b/depparse/pom.xml index d5be7b3c5..e5fd11c7b 100644 --- a/depparse/pom.xml +++ b/depparse/pom.xml @@ -7,7 +7,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 illinois-depparse @@ -16,27 +16,27 @@ edu.illinois.cs.cogcomp illinois-core-utilities - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-edison - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-lemmatizer - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-pos - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-chunker - 4.0.11 + 4.0.12 diff --git a/edison/pom.xml b/edison/pom.xml index 74fb9e717..8e65d94a1 100644 --- a/edison/pom.xml +++ b/edison/pom.xml @@ -7,7 +7,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 illinois-edison @@ -16,7 +16,7 @@ edu.illinois.cs.cogcomp illinois-core-utilities - 4.0.11 + 4.0.12 @@ -80,13 +80,13 @@ edu.illinois.cs.cogcomp illinois-corpusreaders - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-curator - 4.0.11 + 4.0.12 test diff --git a/external/clausie/pom.xml b/external/clausie/pom.xml index e3d68af20..214d4d137 100644 --- a/external/clausie/pom.xml +++ b/external/clausie/pom.xml @@ -5,7 +5,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 ../../pom.xml @@ -24,7 +24,7 @@ edu.illinois.cs.cogcomp external-commons - 4.0.11 + 4.0.12 org.slf4j diff --git a/external/external-commons/pom.xml b/external/external-commons/pom.xml index 33fbd1609..c8ee868ed 100644 --- a/external/external-commons/pom.xml +++ b/external/external-commons/pom.xml @@ -2,7 +2,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 ../../pom.xml @@ -16,12 +16,12 @@ edu.illinois.cs.cogcomp illinois-core-utilities - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-tokenizer - 4.0.11 + 4.0.12 org.cogcomp diff --git a/external/path-lstm/pom.xml b/external/path-lstm/pom.xml index a6eabadab..223ff6705 100644 --- a/external/path-lstm/pom.xml +++ b/external/path-lstm/pom.xml @@ -2,7 +2,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 ../../pom.xml @@ -16,12 +16,12 @@ edu.illinois.cs.cogcomp external-commons - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-edison - 4.0.11 + 4.0.12 org.cogcomp diff --git a/external/stanford_3.3.1/pom.xml b/external/stanford_3.3.1/pom.xml index b0263e8dd..86d5765bc 100644 --- a/external/stanford_3.3.1/pom.xml +++ b/external/stanford_3.3.1/pom.xml @@ -5,7 +5,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 ../../pom.xml @@ -19,7 +19,7 @@ edu.illinois.cs.cogcomp illinois-core-utilities - 4.0.11 + 4.0.12 @@ -36,7 +36,7 @@ edu.illinois.cs.cogcomp illinois-corpusreaders - 4.0.11 + 4.0.12 diff --git a/external/stanford_3.8.0/pom.xml b/external/stanford_3.8.0/pom.xml index 975ffebec..b3c0352a0 100644 --- a/external/stanford_3.8.0/pom.xml +++ b/external/stanford_3.8.0/pom.xml @@ -2,7 +2,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 ../../pom.xml @@ -16,12 +16,12 @@ edu.illinois.cs.cogcomp illinois-corpusreaders - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp external-commons - 4.0.11 + 4.0.12 org.slf4j diff --git a/inference/pom.xml b/inference/pom.xml index 7503b6f06..824130ddd 100644 --- a/inference/pom.xml +++ b/inference/pom.xml @@ -6,7 +6,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 jar @@ -22,7 +22,7 @@ edu.illinois.cs.cogcomp illinois-core-utilities - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp diff --git a/lbjava-nlp-tools/pom.xml b/lbjava-nlp-tools/pom.xml index adb8ac5b3..5e706b556 100644 --- a/lbjava-nlp-tools/pom.xml +++ b/lbjava-nlp-tools/pom.xml @@ -4,7 +4,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 LBJava-NLP-tools @@ -30,7 +30,7 @@ edu.illinois.cs.cogcomp illinois-core-utilities - 4.0.11 + 4.0.12 org.slf4j diff --git a/lemmatizer/pom.xml b/lemmatizer/pom.xml index 4f9d290a7..533fef4f5 100644 --- a/lemmatizer/pom.xml +++ b/lemmatizer/pom.xml @@ -7,7 +7,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 illinois-lemmatizer @@ -16,12 +16,12 @@ edu.illinois.cs.cogcomp illinois-core-utilities - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-edison - 4.0.11 + 4.0.12 edu.stanford.nlp diff --git a/md/pom.xml b/md/pom.xml index 6b331db41..60d07dbfa 100644 --- a/md/pom.xml +++ b/md/pom.xml @@ -3,7 +3,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 4.0.0 @@ -25,32 +25,32 @@ edu.illinois.cs.cogcomp illinois-corpusreaders - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-core-utilities - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-pos - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-edison - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-ner - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp stanford_3.3.1 - 4.0.11 + 4.0.12 org.slf4j diff --git a/ner/config/log4j.properties b/ner/config/log4j.properties new file mode 100644 index 000000000..63cba917e --- /dev/null +++ b/ner/config/log4j.properties @@ -0,0 +1,9 @@ +# Set root logger level to DEBUG and its only appender to A1. +log4j.rootLogger=INFO, A1 + +# A1 is set to be a ConsoleAppender. +log4j.appender.A1=org.apache.log4j.ConsoleAppender + +# A1 uses PatternLayout. +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n diff --git a/ner/pom.xml b/ner/pom.xml index 31daf1cd4..fd8733f09 100644 --- a/ner/pom.xml +++ b/ner/pom.xml @@ -6,7 +6,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 illinois-ner @@ -23,12 +23,12 @@ edu.illinois.cs.cogcomp illinois-tokenizer - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-core-utilities - 4.0.11 + 4.0.12 org.cogcomp @@ -44,7 +44,7 @@ edu.illinois.cs.cogcomp LBJava-NLP-tools - 4.0.11 + 4.0.12 org.slf4j diff --git a/ner/scripts/benchmarks.sh b/ner/scripts/benchmarks.sh index d02fedd0b..f05794870 100755 --- a/ner/scripts/benchmarks.sh +++ b/ner/scripts/benchmarks.sh @@ -30,7 +30,7 @@ if [ ! -e $LIB ]; then fi # Classpath -cpath=".:target/classes" +cpath="config:.:target/classes" for JAR in `ls $DIST/*.jar`; do cpath="$cpath:$JAR" diff --git a/ner/scripts/runNER.sh b/ner/scripts/runNER.sh index 881ff3ac1..054d75e54 100755 --- a/ner/scripts/runNER.sh +++ b/ner/scripts/runNER.sh @@ -34,5 +34,5 @@ for JAR in `ls $LIB/*jar`; do cpath="$cpath:$JAR" done -CMD="java -classpath ${cpath} -Xms3g -Xmx10g -agentpath:/Applications/YourKit-Java-Profiler-2017.02.app/Contents/Resources/bin/mac/libyjpagent.jnilib edu.illinois.cs.cogcomp.ner.Main $1 $2" +CMD="java -classpath ${cpath} -Xms3g -Xmx10g edu.illinois.cs.cogcomp.ner.Main $1 $2" $CMD diff --git a/ner/src/main/java/edu/illinois/cs/cogcomp/ner/LbjTagger/LearningCurveMultiDataset.java b/ner/src/main/java/edu/illinois/cs/cogcomp/ner/LbjTagger/LearningCurveMultiDataset.java index a795277b6..b8dee1ccc 100644 --- a/ner/src/main/java/edu/illinois/cs/cogcomp/ner/LbjTagger/LearningCurveMultiDataset.java +++ b/ner/src/main/java/edu/illinois/cs/cogcomp/ner/LbjTagger/LearningCurveMultiDataset.java @@ -25,6 +25,11 @@ import java.io.IOException; import java.util.Vector; +/** + * build the model test it out, report results and so on. + * @author nick + * @author redman + */ public class LearningCurveMultiDataset { private static final String NAME = LearningCurveMultiDataset.class.getCanonicalName(); @@ -41,7 +46,7 @@ public class LearningCurveMultiDataset { * @param devDataPath data used to auto-converge. */ public static void buildFinalModel(int fixedNumIterations, String trainDataPath, - String testDataPath, String devDataPath) throws Exception { + String testDataPath, String devDataPath, boolean incremental) throws Exception { Data trainData = new Data(trainDataPath, trainDataPath, "-c", new String[] {}, new String[] {}); ExpressiveFeaturesAnnotator.annotate(trainData); Data testData = new Data(testDataPath, testDataPath, "-c", new String[] {}, new String[] {}); @@ -55,19 +60,20 @@ public static void buildFinalModel(int fixedNumIterations, String trainDataPath, test.addElement(devData); logger.debug("Building final model: iterations = " + fixedNumIterations + " train = '" + trainDataPath + "' test = '"+testDataPath+"' dev = '" + testDataPath+"'"); - getLearningCurve(train, test, fixedNumIterations); + getLearningCurve(train, test, fixedNumIterations, incremental); } /** * Convenience function that has a default value for dataFormat of -c - * @param fixedNumIterations - * @param trainDataPath - * @param testDataPath + * @param fixedNumIterations if this is > -1 the number of training iterations that will run. + * @param trainDataPath the path on the file system for the training data. + * @param testDataPath the path on the file system for the test data used to test convergence. + * @param incremental if the model is being incremented, this is true. * @throws Exception */ public static void getLearningCurve(int fixedNumIterations, String trainDataPath, - String testDataPath) throws Exception { - getLearningCurve(fixedNumIterations, trainDataPath, "-c", testDataPath); + String testDataPath, boolean incremental) throws Exception { + getLearningCurve(fixedNumIterations, trainDataPath, "-c", testDataPath, incremental); } /** @@ -78,9 +84,15 @@ public static void getLearningCurve(int fixedNumIterations, String trainDataPath *

* In practice, testDataPath should be a Development set. *

+ * @param fixedNumIterations if this is > -1 the number of training iterations that will run. + * @param dataFormat the data format, bracketed or column. + * @param trainDataPath the path on the file system for the training data. + * @param testDataPath the path on the file system for the test data used to test convergence. + * @param incremental if the model is being incremented, this is true. + * @throws Exception */ public static void getLearningCurve(int fixedNumIterations, String dataFormat, String trainDataPath, - String testDataPath) throws Exception { + String testDataPath, boolean incremental) throws Exception { logger.debug("getLearningCurve(): fni = " + fixedNumIterations + " trainDataPath = '" + trainDataPath + "' testDataPath = '" + testDataPath + "'...."); Data trainData = @@ -93,16 +105,22 @@ public static void getLearningCurve(int fixedNumIterations, String dataFormat, S train.addElement(trainData); Vector test = new Vector<>(); test.addElement(testData); - getLearningCurve(train, test, fixedNumIterations); + getLearningCurve(train, test, fixedNumIterations, incremental); } /** - * use fixedNumIterations=-1 if you want to use the automatic convergence criterion + * use fixedNumIterations=-1 if you want to use the automatic convergence criterion, incremental + * true will start with the existing models weights, and continue training with that set of default + * weights. Training data is assumed to be in column format. *

- * NB: assuming column format + * @param fixedNumIterations if this is > -1 the number of training iterations that will run. + * @param trainDataSet the path on the file system for the training data. + * @param testDataSet the path on the file system for the test data used to test convergence. + * @param incremental if the model is being incremented, this is true. + * @throws Exception */ public static void getLearningCurve(Vector trainDataSet, Vector testDataSet, - int fixedNumIterations) throws Exception { + int fixedNumIterations, boolean incremental) throws Exception { double bestF1Level1 = -1; int bestRoundLevel1 = 0; // Get the directory name (.model is appended in LbjTagger/Parameters.java:139) @@ -129,9 +147,15 @@ public static void getLearningCurve(Vector trainDataSet, Vector test paramLevel1.baseLTU.featurePruningThreshold = ParametersForLbjCode.currentParameters.featurePruningThreshold; logger.info("Level 1 classifier learning rate = "+ParametersForLbjCode.currentParameters.learningRatePredictionsLevel1+ ", thickness = "+ParametersForLbjCode.currentParameters.thicknessPredictionsLevel1); + NETaggerLevel1 tagger1 = new NETaggerLevel1(paramLevel1, modelPath + ".level1", modelPath + ".level1.lex"); - tagger1.forget(); + if (!incremental) { + logger.info("Training L1 model from scratch."); + tagger1.forget(); + } else { + logger.info("Training L1 model incrementally."); + } ParametersForLbjCode.currentParameters.taggerLevel1 = tagger1; for (int dataId = 0; dataId < trainDataSet.size(); dataId++) { Data trainData = trainDataSet.elementAt(dataId); @@ -208,7 +232,12 @@ public static void getLearningCurve(Vector trainDataSet, Vector test new NETaggerLevel2(paramLevel2, ParametersForLbjCode.currentParameters.pathToModelFile + ".level2", ParametersForLbjCode.currentParameters.pathToModelFile + ".level2.lex"); - tagger2.forget(); + if (!incremental) { + logger.info("Training L2 model from scratch."); + tagger2.forget(); + } else { + logger.info("Training L2 model incrementally."); + } ParametersForLbjCode.currentParameters.taggerLevel2 = tagger2; // Previously checked if PatternFeatures was in featuresToUse. diff --git a/ner/src/main/java/edu/illinois/cs/cogcomp/ner/Main.java b/ner/src/main/java/edu/illinois/cs/cogcomp/ner/Main.java index 111f9f5df..1b4f8e638 100644 --- a/ner/src/main/java/edu/illinois/cs/cogcomp/ner/Main.java +++ b/ner/src/main/java/edu/illinois/cs/cogcomp/ner/Main.java @@ -248,14 +248,18 @@ protected void inputMenu() { System.out.println("Loading resources..."); if (resourceManager == null) this.resourceManager = new NerBaseConfigurator().getDefaultConfig(); - - String modelName = this.resourceManager.getString(NerBaseConfigurator.MODEL_NAME); - if (modelName.toLowerCase().equals("conll")) - this.nerAnnotator = new NERAnnotator(this.resourceManager, ViewNames.NER_CONLL); - else if (modelName.toLowerCase().equals("ontonotes")) - this.nerAnnotator = new NERAnnotator(this.resourceManager, ViewNames.NER_ONTONOTES); - else - this.nerAnnotator = new NERAnnotator(this.resourceManager, "NER_OTHER"); + String viewName = this.resourceManager.getString(NerBaseConfigurator.VIEW_NAME); + if (viewName == null) { + String modelName = this.resourceManager.getString(NerBaseConfigurator.MODEL_NAME); + if (modelName.toLowerCase().equals("conll")) + this.nerAnnotator = new NERAnnotator(this.resourceManager, ViewNames.NER_CONLL); + else if (modelName.toLowerCase().equals("ontonotes")) + this.nerAnnotator = new NERAnnotator(this.resourceManager, ViewNames.NER_ONTONOTES); + else + this.nerAnnotator = new NERAnnotator(this.resourceManager, "NER_OTHER"); + } else { + this.nerAnnotator = new NERAnnotator(this.resourceManager, viewName); + } System.out.println("Completed loading resources, assuming a "); } diff --git a/ner/src/main/java/edu/illinois/cs/cogcomp/ner/ModelLoader.java b/ner/src/main/java/edu/illinois/cs/cogcomp/ner/ModelLoader.java index 37337fed5..a968543a1 100644 --- a/ner/src/main/java/edu/illinois/cs/cogcomp/ner/ModelLoader.java +++ b/ner/src/main/java/edu/illinois/cs/cogcomp/ner/ModelLoader.java @@ -37,9 +37,10 @@ public class ModelLoader { * Load the models wherever they are found. Check file system first, then classpath, and finally get it * from Minio datastore. * @param rm the resource manager. + * @param training if we are training. * @param viewName the name of the view identifies the model. */ - static public void load(ResourceManager rm, String viewName) { + static public void load(ResourceManager rm, String viewName, boolean training) { // the loaded built into the model will check the local file system and the jar files in the classpath. ParametersForLbjCode cp = ParametersForLbjCode.currentParameters; @@ -65,6 +66,18 @@ static public void load(ResourceManager rm, String viewName) { } else { logger.info("L2 model not required."); } + } else if (training) { + + // we are training a new model, so it it doesn't exist, we don't care, just create a + // container. + tagger1 = new NETaggerLevel1(modelPath + ".level1", modelPath + ".level1.lex"); + logger.info("Reading L1 model from file : " + modelPath + ".level2"); + if (cp.featuresToUse.containsKey("PredictionsLevel1")) { + tagger2 = new NETaggerLevel2(modelPath + ".level2", modelPath + ".level2.lex"); + logger.info("Reading L2 model from file : " + modelPath + ".level2"); + } else { + logger.info("L2 model not required."); + } } else { // all else has filed, load from the datastore, create artifact ids based on the view diff --git a/ner/src/main/java/edu/illinois/cs/cogcomp/ner/NERAnnotator.java b/ner/src/main/java/edu/illinois/cs/cogcomp/ner/NERAnnotator.java index 5edc4c96f..977b98ccc 100644 --- a/ner/src/main/java/edu/illinois/cs/cogcomp/ner/NERAnnotator.java +++ b/ner/src/main/java/edu/illinois/cs/cogcomp/ner/NERAnnotator.java @@ -25,6 +25,7 @@ import edu.illinois.cs.cogcomp.core.utilities.configuration.Configurator; import edu.illinois.cs.cogcomp.core.utilities.configuration.ResourceManager; import edu.illinois.cs.cogcomp.lbjava.learn.Lexicon; +import edu.illinois.cs.cogcomp.lbjava.learn.SparseNetworkLearner; import edu.illinois.cs.cogcomp.lbjava.parse.LinkedVector; import edu.illinois.cs.cogcomp.ner.ExpressiveFeatures.ExpressiveFeaturesAnnotator; import edu.illinois.cs.cogcomp.ner.InferenceMethods.Decoder; @@ -47,6 +48,12 @@ public class NERAnnotator extends Annotator { /** our specific logger. */ private final Logger logger = LoggerFactory.getLogger(NERAnnotator.class); + /** the level one model. */ + public SparseNetworkLearner taggerLevel1; + + /** the level two model. */ + public SparseNetworkLearner taggerLevel2; + /** * @param nonDefaultConfigValues a configuration file specifying non-default parameters for the * NER model to use @@ -82,7 +89,9 @@ public NERAnnotator(ResourceManager nonDefaultRm, String viewName) { AnnotatorConfigurator.IS_LAZILY_INITIALIZED.key, Configurator.TRUE), nonDefaultRm); } - + /** this is used to sync loading models. */ + static final String LOADING_MODELS = "LOADING_MODELS"; + /** * Superclass calls this method either on instantiation or at first call to getView(). Logging * has been disabled because non-static logger is not initialized at the time this is called if @@ -102,7 +111,11 @@ public void initialize(ResourceManager nerRm) { Parameters.readConfigAndLoadExternalData(nerRm); // load the models. - ModelLoader.load(nerRm, viewName); + synchronized (LOADING_MODELS) { + ModelLoader.load(nerRm, viewName, false); + this.taggerLevel1 = ParametersForLbjCode.currentParameters.taggerLevel1; + this.taggerLevel2 = ParametersForLbjCode.currentParameters.taggerLevel2; + } } /** @@ -139,8 +152,8 @@ public void addView(TextAnnotation ta) { Data data = new Data(new NERDocument(sentences, "input")); try { ExpressiveFeaturesAnnotator.annotate(data); - Decoder.annotateDataBIO(data, (NETaggerLevel1) ParametersForLbjCode.currentParameters.taggerLevel1, - (NETaggerLevel2) ParametersForLbjCode.currentParameters.taggerLevel2); + Decoder.annotateDataBIO(data, (NETaggerLevel1) taggerLevel1, + (NETaggerLevel2) taggerLevel2); } catch (Exception e) { logger.error("Cannot annotate the text, the exception was: ", e); return; @@ -228,7 +241,7 @@ public Set getTagValues() { if (!isInitialized()) { doInitialize(); } - Lexicon labelLexicon = ParametersForLbjCode.currentParameters.taggerLevel1.getLabelLexicon(); + Lexicon labelLexicon = taggerLevel1.getLabelLexicon(); Set tagSet = new HashSet(); for (int i =0; i < labelLexicon.size(); ++i) { tagSet.add(labelLexicon.lookupKey(i).getStringValue()); diff --git a/ner/src/main/java/edu/illinois/cs/cogcomp/ner/NerBenchmark.java b/ner/src/main/java/edu/illinois/cs/cogcomp/ner/NerBenchmark.java index 5a00e58b2..fe16d315f 100644 --- a/ner/src/main/java/edu/illinois/cs/cogcomp/ner/NerBenchmark.java +++ b/ner/src/main/java/edu/illinois/cs/cogcomp/ner/NerBenchmark.java @@ -8,8 +8,10 @@ package edu.illinois.cs.cogcomp.ner; import edu.illinois.cs.cogcomp.core.utilities.configuration.ResourceManager; +import edu.illinois.cs.cogcomp.lbjava.learn.SparseAveragedPerceptron; +import edu.illinois.cs.cogcomp.ner.LbjFeatures.NETaggerLevel1; +import edu.illinois.cs.cogcomp.ner.LbjFeatures.NETaggerLevel2; import edu.illinois.cs.cogcomp.ner.LbjTagger.*; -import edu.illinois.cs.cogcomp.ner.config.NerBaseConfigurator; import java.io.File; import java.io.FilenameFilter; @@ -44,6 +46,7 @@ * -features : for debugging, reports the feature vector for each token in the dataset. Output produced in a "features.out" file. * -iterations : specify a fixed number of iterations, or -1 (the default) means auto converge requiring a "dev" directory. * -release : build a final model for release, it will build on test and train, and unless "-iterations" specified, it will autoconvert + * -incremental : rather than discarding existing weights, start with those that already exist and continue training. * using "dev" for a holdout set. * } */ @@ -75,6 +78,10 @@ public class NerBenchmark { /** -1 to converge automatically, positive number to do a fixed number of iterations. */ static int iterations = -1; + + /** If this is set, we will start with the existing weights (and averages) for the + * model and continue training from there. */ + static boolean incremental = false; /** * parse the arguments, only the directory. @@ -94,6 +101,10 @@ private static void parseArguments(String[] args) { } directory = args[i]; break; + case "-incremental": + System.out.println("Configured for incremental training."); + incremental = true; + break; case "-verbose": verbose = true; break; @@ -182,13 +193,25 @@ public boolean accept(File dir, String name) { if (!skiptraining) { if (trainDir.exists() && testDir.exists() && devDir.exists()) { System.out.println("\n\n----- Training models for evaluation for "+confFile+" ------"); - Parameters.readConfigAndLoadExternalData(confFile, !skiptraining); + Parameters.readConfigAndLoadExternalData(confFile, true); ResourceManager rm = new ResourceManager(confFile); - ModelLoader.load(rm, rm.getString("modelName")); + ModelLoader.load(rm, rm.getString("modelName"), true); + NETaggerLevel1 taggerLevel1 = (NETaggerLevel1) ParametersForLbjCode.currentParameters.taggerLevel1; + NETaggerLevel2 taggerLevel2 = (NETaggerLevel2) ParametersForLbjCode.currentParameters.taggerLevel2; + SparseAveragedPerceptron sap1 = (SparseAveragedPerceptron)taggerLevel1.getBaseLTU(); + sap1.setLearningRate(ParametersForLbjCode.currentParameters.learningRatePredictionsLevel1); + sap1.setThickness(ParametersForLbjCode.currentParameters.thicknessPredictionsLevel1); + System.out.println("L1 learning rate = "+sap1.getLearningRate()+", thickness = "+sap1.getPositiveThickness()); + if (ParametersForLbjCode.currentParameters.featuresToUse.containsKey("PredictionsLevel1")) { + SparseAveragedPerceptron sap2 = (SparseAveragedPerceptron)taggerLevel2.getBaseLTU(); + sap2.setLearningRate(ParametersForLbjCode.currentParameters.learningRatePredictionsLevel2); + sap2.setThickness(ParametersForLbjCode.currentParameters.thicknessPredictionsLevel2); + System.out.println("L2 learning rate = "+sap2.getLearningRate()+", thickness = "+sap2.getPositiveThickness()); + } // there is a training directory, with training enabled, so train. We use the same dataset // for both training and evaluating. - LearningCurveMultiDataset.getLearningCurve(iterations, trainDirName, devDirName); + LearningCurveMultiDataset.getLearningCurve(iterations, trainDirName, devDirName, incremental); System.out.println("\n\n----- Final results for "+confFile+", verbose ------"); NETesterMultiDataset.test(testDirName, true, ParametersForLbjCode.currentParameters.labelsToIgnoreInEvaluation, @@ -205,7 +228,7 @@ public boolean accept(File dir, String name) { System.out.println("\n\n----- Reporting results from existing models for "+confFile+" ------"); Parameters.readConfigAndLoadExternalData(confFile, !skiptraining); ResourceManager rm = new ResourceManager(confFile); - ModelLoader.load(rm, rm.getString("modelName")); + ModelLoader.load(rm, rm.getString("modelName"), !skiptraining); System.out.println("Benchmark against configuration : " + confFile); if (reportLabels) NEDisplayPredictions.test(testDirName, "-c", verbose); @@ -219,14 +242,14 @@ else if (reportFeatures) if (release) { if (trainDir.exists() && testDir.exists() && devDir.exists()) { - Parameters.readConfigAndLoadExternalData(confFile, !skiptraining); + Parameters.readConfigAndLoadExternalData(confFile, true); ResourceManager rm = new ResourceManager(confFile); - ModelLoader.load(rm, rm.getString("modelName")); + ModelLoader.load(rm, rm.getString("modelName"), true); System.out.println("\n\n----- Building a final model for "+confFile+" ------"); // there is a training directory, with training enabled, so train. We use the same dataset // for both training and evaluating. - LearningCurveMultiDataset.buildFinalModel(iterations, trainDirName, testDirName, devDirName); + LearningCurveMultiDataset.buildFinalModel(iterations, trainDirName, testDirName, devDirName, incremental); System.out.println("\n\n----- Release results for "+confFile+", verbose ------"); NETesterMultiDataset.test(devDirName, true, ParametersForLbjCode.currentParameters.labelsToIgnoreInEvaluation, diff --git a/ner/src/main/java/edu/illinois/cs/cogcomp/ner/NerTagger.java b/ner/src/main/java/edu/illinois/cs/cogcomp/ner/NerTagger.java index cb4e199be..4e1280718 100644 --- a/ner/src/main/java/edu/illinois/cs/cogcomp/ner/NerTagger.java +++ b/ner/src/main/java/edu/illinois/cs/cogcomp/ner/NerTagger.java @@ -36,7 +36,6 @@ public static void main(String[] args) { boolean areWeTraining = args[0].equalsIgnoreCase("-train"); ResourceManager rm = new ResourceManager(args[args.length - 1]); Parameters.readConfigAndLoadExternalData(args[args.length - 1], areWeTraining); - if (args[0].equalsIgnoreCase("-train")) { String dataFormat; // config file is always the last one. @@ -45,13 +44,12 @@ public static void main(String[] args) { }else{ dataFormat = args[3]; } - LearningCurveMultiDataset.getLearningCurve(-1, dataFormat, args[1], args[2]); + LearningCurveMultiDataset.getLearningCurve(-1, dataFormat, args[1], args[2], false); }else if (args[0].equalsIgnoreCase("-trainFixedIterations")) - LearningCurveMultiDataset.getLearningCurve(Integer.parseInt(args[1]), args[2], - args[3]); + LearningCurveMultiDataset.getLearningCurve(Integer.parseInt(args[1]), args[2], args[3], false); else { // load up the models - ModelLoader.load(rm, rm.getString("modelName")); + ModelLoader.load(rm, rm.getString("modelName"), false); if (args[0].equalsIgnoreCase("-annotate")) { NETagPlain.init(); NETagPlain.tagData(args[1], args[2]); @@ -85,7 +83,8 @@ public static void main(String[] args) { } NETesterMultiDataset.test(args[1], true, dataFormat, cp.labelsToIgnoreInEvaluation, cp.labelsToAnonymizeInEvaluation); - }if (args[0].equalsIgnoreCase("-dumpFeatures")) + } + if (args[0].equalsIgnoreCase("-dumpFeatures")) NETesterMultiDataset.dumpFeaturesLabeledData(args[1], args[2]); } } catch (Exception e) { diff --git a/ner/src/test/java/edu/illinois/cs/cogcomp/ner/NerTest.java b/ner/src/test/java/edu/illinois/cs/cogcomp/ner/NerTest.java index 462b6470f..514944915 100644 --- a/ner/src/test/java/edu/illinois/cs/cogcomp/ner/NerTest.java +++ b/ner/src/test/java/edu/illinois/cs/cogcomp/ner/NerTest.java @@ -44,7 +44,7 @@ public void setUp() throws Exception { ResourceManager rm = new NerBaseConfigurator().getDefaultConfig(); Parameters.readConfigAndLoadExternalData(rm); ParametersForLbjCode.currentParameters.forceNewSentenceOnLineBreaks = false; - ModelLoader.load(rm, ViewNames.NER_ONTONOTES); + ModelLoader.load(rm, ViewNames.NER_ONTONOTES, false); t1 = (NETaggerLevel1) ParametersForLbjCode.currentParameters.taggerLevel1; t2 = (NETaggerLevel2) ParametersForLbjCode.currentParameters.taggerLevel2; } catch (Exception e) { diff --git a/pipeline-client/pom.xml b/pipeline-client/pom.xml index 8b19284c2..37461f1ec 100644 --- a/pipeline-client/pom.xml +++ b/pipeline-client/pom.xml @@ -6,7 +6,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 illinois-pipeline-client @@ -15,7 +15,7 @@ edu.illinois.cs.cogcomp illinois-core-utilities - 4.0.11 + 4.0.12 diff --git a/pipeline/pom.xml b/pipeline/pom.xml index f0ba6d199..5424d64ba 100644 --- a/pipeline/pom.xml +++ b/pipeline/pom.xml @@ -5,7 +5,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 illinois-nlp-pipeline @@ -16,57 +16,57 @@ edu.illinois.cs.cogcomp illinois-core-utilities - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-tokenizer - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-chunker - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-lemmatizer - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-corpusreaders - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-quantifier - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-prep-srl - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-comma - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-verbsense - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-question-typer - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp stanford_3.3.1 - 4.0.11 + 4.0.12 org.cogcomp @@ -83,7 +83,7 @@ edu.illinois.cs.cogcomp illinois-ner - 4.0.11 + 4.0.12 org.apache.commons @@ -93,17 +93,17 @@ edu.illinois.cs.cogcomp illinois-md - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-relation-extraction - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-datalessclassification - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp @@ -131,7 +131,7 @@ edu.illinois.cs.cogcomp illinois-depparse - 4.0.11 + 4.0.12 @@ -149,12 +149,12 @@ edu.illinois.cs.cogcomp illinois-time - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-transliteration - 4.0.11 + 4.0.12 diff --git a/pom.xml b/pom.xml index fc684221b..e5db3763c 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ edu.illinois.cs.cogcomp illinois-cogcomp-nlp pom - 4.0.11 + 4.0.12 core-utilities tokenizer diff --git a/pos/pom.xml b/pos/pom.xml index 63dbe8ac1..f10f81edb 100644 --- a/pos/pom.xml +++ b/pos/pom.xml @@ -4,7 +4,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 illinois-pos diff --git a/prepsrl/pom.xml b/prepsrl/pom.xml index 0a30dd267..dd2969f0c 100644 --- a/prepsrl/pom.xml +++ b/prepsrl/pom.xml @@ -5,7 +5,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 4.0.0 @@ -15,32 +15,32 @@ edu.illinois.cs.cogcomp illinois-core-utilities - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-edison - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp LBJava-NLP-tools - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-depparse - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-pos - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-lemmatizer - 4.0.11 + 4.0.12 diff --git a/quantifier/pom.xml b/quantifier/pom.xml index 0d6072ff5..bc00e35d3 100644 --- a/quantifier/pom.xml +++ b/quantifier/pom.xml @@ -4,7 +4,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 illinois-quantifier @@ -35,31 +35,31 @@ edu.illinois.cs.cogcomp illinois-core-utilities - 4.0.11 + 4.0.12 compile edu.illinois.cs.cogcomp illinois-tokenizer - 4.0.11 + 4.0.12 compile edu.illinois.cs.cogcomp illinois-pos - 4.0.11 + 4.0.12 compile edu.illinois.cs.cogcomp illinois-edison - 4.0.11 + 4.0.12 compile edu.illinois.cs.cogcomp illinois-curator - 4.0.11 + 4.0.12 compile diff --git a/question-type/pom.xml b/question-type/pom.xml index 3a17b1124..7921f446a 100644 --- a/question-type/pom.xml +++ b/question-type/pom.xml @@ -4,7 +4,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 illinois-question-typer @@ -13,22 +13,22 @@ edu.illinois.cs.cogcomp illinois-core-utilities - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-edison - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-tokenizer - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-pipeline-client - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp diff --git a/relation-extraction/pom.xml b/relation-extraction/pom.xml index eef25ffc2..8cd175d58 100644 --- a/relation-extraction/pom.xml +++ b/relation-extraction/pom.xml @@ -3,7 +3,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 4.0.0 @@ -33,42 +33,42 @@ edu.illinois.cs.cogcomp illinois-corpusreaders - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-core-utilities - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-pos - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp stanford_3.3.1 - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-edison - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-ner - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-md - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-chunker - 4.0.11 + 4.0.12 joda-time diff --git a/similarity/pom.xml b/similarity/pom.xml index c837306e3..de7691b62 100644 --- a/similarity/pom.xml +++ b/similarity/pom.xml @@ -4,7 +4,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 illinois-similarity @@ -13,7 +13,7 @@ edu.illinois.cs.cogcomp illinois-core-utilities - 4.0.11 + 4.0.12 org.slf4j @@ -55,7 +55,7 @@ edu.illinois.cs.cogcomp illinois-tokenizer - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp diff --git a/temporal-normalizer/pom.xml b/temporal-normalizer/pom.xml index 7d696d43b..a7678bb8f 100644 --- a/temporal-normalizer/pom.xml +++ b/temporal-normalizer/pom.xml @@ -5,7 +5,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 4.0.0 illinois-time @@ -13,7 +13,7 @@ edu.illinois.cs.cogcomp illinois-core-utilities - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp @@ -28,7 +28,7 @@ edu.illinois.cs.cogcomp illinois-pos - 4.0.11 + 4.0.12 org.apache.uima @@ -43,12 +43,12 @@ edu.illinois.cs.cogcomp illinois-chunker - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-curator - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp @@ -58,7 +58,7 @@ edu.illinois.cs.cogcomp illinois-ner - 4.0.11 + 4.0.12 test @@ -86,7 +86,7 @@ edu.illinois.cs.cogcomp illinois-tokenizer - 4.0.11 + 4.0.12 diff --git a/tokenizer/pom.xml b/tokenizer/pom.xml index e4095102b..683f9f469 100644 --- a/tokenizer/pom.xml +++ b/tokenizer/pom.xml @@ -6,7 +6,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 illinois-tokenizer @@ -15,17 +15,17 @@ edu.illinois.cs.cogcomp illinois-core-utilities - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp LBJava-NLP-tools - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-curator - 4.0.11 + 4.0.12 test diff --git a/transliteration/pom.xml b/transliteration/pom.xml index d9fde8cc6..04fb63691 100644 --- a/transliteration/pom.xml +++ b/transliteration/pom.xml @@ -5,7 +5,7 @@ http://www.w3.org/2001/XMLSchema-instance "> illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 4.0.0 @@ -20,12 +20,12 @@ http://www.w3.org/2001/XMLSchema-instance "> edu.illinois.cs.cogcomp illinois-core-utilities - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-edison - 4.0.11 + 4.0.12 org.apache.commons diff --git a/verbsense/pom.xml b/verbsense/pom.xml index 812618517..80b1285b2 100755 --- a/verbsense/pom.xml +++ b/verbsense/pom.xml @@ -3,7 +3,7 @@ illinois-cogcomp-nlp edu.illinois.cs.cogcomp - 4.0.11 + 4.0.12 4.0.0 illinois-verbsense @@ -18,37 +18,37 @@ edu.illinois.cs.cogcomp illinois-edison - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-core-utilities - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-pos - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-ner - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-chunker - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-lemmatizer - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp illinois-inference - 4.0.11 + 4.0.12 edu.illinois.cs.cogcomp