Merge branch 'master' of jamie.stanford.edu:/u/nlp/git/javanlp

Author: acvogel

src/edu/stanford/nlp/ie/AbstractSequenceClassifier.java

@@ -87,8 +87,8 @@ public abstract class AbstractSequenceClassifier<IN extends CoreMap> implements
   private CoreTokenFactory<IN> tokenFactory;
   public int windowSize;
   // different threads can add or query knownLCWords at the same time,
-  // so we need a concurrent data structure
-  protected Set<String> knownLCWords = Collections.newSetFromMap(new ConcurrentHashMap<String,Boolean>());
+  // so we need a concurrent data structure.  created in reinit()
+  protected Set<String> knownLCWords = null;
 
   private DocumentReaderAndWriter<IN> defaultReaderAndWriter;
   public DocumentReaderAndWriter<IN> defaultReaderAndWriter() {
@@ -165,6 +165,12 @@ protected final void reinit() {
     } else {
       plainTextReaderAndWriter = makePlainTextReaderAndWriter();
     }
+
+    if (!flags.useKnownLCWords) {
+      knownLCWords = Collections.emptySet();
+    } else if (knownLCWords == null || knownLCWords.size() == 0) {
+      knownLCWords = Collections.newSetFromMap(new ConcurrentHashMap<String,Boolean>());
+    }
   }
 
   /**

src/edu/stanford/nlp/sequences/ObjectBankWrapper.java

@@ -136,7 +136,7 @@ private void doBasicStuff(List<IN> doc) {
         if (flags.wordFunction != null) {
           word = flags.wordFunction.apply(word);
         }
-        if (word.length() > 0) {
+        if (flags.useKnownLCWords && word.length() > 0) {
           char ch = word.charAt(0);
           if (Character.isLowerCase(ch)) {
             knownLCWords.add(word);

src/edu/stanford/nlp/sequences/SeqClassifierFlags.java

@@ -996,6 +996,7 @@ public class SeqClassifierFlags implements Serializable {
   public boolean useHardGE = false;
   public boolean useCRFforUnsup = false;
   public boolean useGEforSup = false;
+  public boolean useKnownLCWords = true;
 
   // "ADD VARIABLES ABOVE HERE"
 
@@ -2478,7 +2479,8 @@ public void setProperties(Properties props, boolean printProps) {
         useCRFforUnsup = Boolean.parseBoolean(val);
       } else if (key.equalsIgnoreCase("useGEforSup")){
         useGEforSup = Boolean.parseBoolean(val);
-
+      } else if (key.equalsIgnoreCase("useKnownLCWords")){
+        useKnownLCWords = Boolean.parseBoolean(val);
         // ADD VALUE ABOVE HERE
       } else if (key.length() > 0 && !key.equals("prop")) {
         System.err.println("Unknown property: |" + key + '|');