Add an option to turn off ssplit in the tokenizer annotator. Not sure this is useful, but at least it allows for more fine-grained testing of the WordToSentenceProcessor

Author: AngledLuffa

src/edu/stanford/nlp/pipeline/TokenizerAnnotator.java

@@ -256,7 +256,11 @@ public TokenizerAnnotator(boolean verbose, Properties props, String options) {
       this.cleanxmlAnnotator = null;
     }
 
-    this.ssplitAnnotator = new WordsToSentencesAnnotator(props);
+    if (PropertiesUtils.getBool(props, STANFORD_TOKENIZE + "." + STANFORD_SSPLIT, true)) {
+      this.ssplitAnnotator = new WordsToSentencesAnnotator(props);
+    } else {
+      this.ssplitAnnotator = null;
+    }
   }
 
   /**
@@ -445,7 +449,9 @@ public void annotate(Annotation annotation) {
     if (this.cleanxmlAnnotator != null) {
       this.cleanxmlAnnotator.annotate(annotation);
     }
-    this.ssplitAnnotator.annotate(annotation);
+    if (this.ssplitAnnotator != null) {
+      this.ssplitAnnotator.annotate(annotation);
+    }
   }
 
   @Override

test/src/edu/stanford/nlp/process/WordToSentenceProcessorTest.java

@@ -18,9 +18,9 @@
 public class WordToSentenceProcessorTest extends TestCase {
 
   private static final TokenizerAnnotator onelineTokenizer =
-    new TokenizerAnnotator(false, PropertiesUtils.asProperties("tokenize.language", "en", "ssplit.isOneSentence", "true"), null);
+    new TokenizerAnnotator(false, PropertiesUtils.asProperties("tokenize.language", "en", "tokenize.ssplit", "false"), null);
   private static final TokenizerAnnotator udNL =
-    new TokenizerAnnotator(false, "en", "invertible,tokenizeNLs=true");
+    new TokenizerAnnotator(false, PropertiesUtils.asProperties("tokenize.language", "en", "tokenize.ssplit", "false"), "invertible,tokenizeNLs=true");
   private static final TokenizerAnnotator wsNL =
     new TokenizerAnnotator(false, PropertiesUtils.asProperties("tokenize.whitespace", "true", "invertible", "true", "tokenizeNLs", "true"));
 
@@ -208,9 +208,8 @@ public void testBlankLines() {
   public void testExclamationPoint() {
     Annotation annotation = new Annotation("Foo!!");
     onelineTokenizer.annotate(annotation);
-    // the TokenizerAnnotator will add ids by default
     List<CoreLabel> list = annotation.get(CoreAnnotations.TokensAnnotation.class);
-    assertEquals("Wrong double bang", "[Foo-1, !!-2]", list.toString());
+    assertEquals("Wrong double bang", "[Foo, !!]", list.toString());
   }
 
   public void testChinese() {