17
17
18
18
public class WordToSentenceProcessorTest extends TestCase {
19
19
20
- private static final TokenizerAnnotator ud =
21
- new TokenizerAnnotator (false , " en" );
20
+ private static final TokenizerAnnotator onelineTokenizer =
21
+ new TokenizerAnnotator (false , PropertiesUtils . asProperties ( "tokenize.language" , " en", "ssplit.isOneSentence" , "true" ), null );
22
22
private static final TokenizerAnnotator udNL =
23
23
new TokenizerAnnotator (false , "en" , "invertible,tokenizeNLs=true" );
24
24
private static final TokenizerAnnotator wsNL =
@@ -33,7 +33,7 @@ public class WordToSentenceProcessorTest extends TestCase {
33
33
34
34
private static void checkResult (WordToSentenceProcessor <CoreLabel > wts ,
35
35
String testSentence , String ... gold ) {
36
- checkResult (wts , ud , testSentence , gold );
36
+ checkResult (wts , onelineTokenizer , testSentence , gold );
37
37
}
38
38
39
39
private static void checkResult (WordToSentenceProcessor <CoreLabel > wts ,
@@ -207,9 +207,10 @@ public void testBlankLines() {
207
207
208
208
public void testExclamationPoint () {
209
209
Annotation annotation = new Annotation ("Foo!!" );
210
- ud .annotate (annotation );
210
+ onelineTokenizer .annotate (annotation );
211
+ // the TokenizerAnnotator will add ids by default
211
212
List <CoreLabel > list = annotation .get (CoreAnnotations .TokensAnnotation .class );
212
- assertEquals ("Wrong double bang" , "[Foo, !!]" , list .toString ());
213
+ assertEquals ("Wrong double bang" , "[Foo-1 , !!-2 ]" , list .toString ());
213
214
}
214
215
215
216
public void testChinese () {
0 commit comments