File tree Expand file tree Collapse file tree 5 files changed +9
-9
lines changed Expand file tree Collapse file tree 5 files changed +9
-9
lines changed Original file line number Diff line number Diff line change @@ -289,14 +289,14 @@ import edu.stanford.nlp.util.PropertiesUtils;
289
289
if (invertible) {
290
290
String str = prevWordAfter. toString();
291
291
prevWordAfter. setLength(0 );
292
- CoreLabel word = (CoreLabel ) tokenFactory. makeToken(txt, yychar, yylength());
292
+ CoreLabel word = (CoreLabel ) tokenFactory. makeToken(txt, Math . toIntExact( yychar) , yylength());
293
293
word. set(CoreAnnotations . OriginalTextAnnotation . class, originalText);
294
294
word. set(CoreAnnotations . BeforeAnnotation . class, str);
295
295
prevWord. set(CoreAnnotations . AfterAnnotation . class, str);
296
296
prevWord = word;
297
297
return word;
298
298
} else {
299
- return tokenFactory. makeToken(txt, yychar, yylength());
299
+ return tokenFactory. makeToken(txt, Math . toIntExact( yychar) , yylength());
300
300
}
301
301
}
302
302
Original file line number Diff line number Diff line change @@ -253,7 +253,7 @@ import edu.stanford.nlp.util.logging.Redwood;
253
253
254
254
private Object getNext(String txt, String originalText, String annotation) {
255
255
txt = LexerUtils . removeSoftHyphens(txt);
256
- Label w = (Label ) tokenFactory. makeToken(txt, yychar, yylength());
256
+ Label w = (Label ) tokenFactory. makeToken(txt, Math . toIntExact( yychar) , yylength());
257
257
if (invertible || annotation != null ) {
258
258
CoreLabel word = (CoreLabel ) w;
259
259
if (invertible) {
Original file line number Diff line number Diff line change @@ -269,7 +269,7 @@ import edu.stanford.nlp.util.logging.Redwood;
269
269
270
270
private Object getNext(String txt, String originalText, String annotation) {
271
271
txt = LexerUtils . removeSoftHyphens(txt);
272
- Label w = (Label ) tokenFactory. makeToken(txt, yychar, yylength());
272
+ Label w = (Label ) tokenFactory. makeToken(txt, Math . toIntExact( yychar) , yylength());
273
273
if (invertible || annotation != null ) {
274
274
CoreLabel word = (CoreLabel ) w;
275
275
if (invertible) {
Original file line number Diff line number Diff line change @@ -489,14 +489,14 @@ import edu.stanford.nlp.util.logging.Redwood;
489
489
if (invertible) {
490
490
String str = prevWordAfter. toString();
491
491
prevWordAfter. setLength(0 );
492
- CoreLabel word = (CoreLabel ) tokenFactory. makeToken(txt, yychar, yylength());
492
+ CoreLabel word = (CoreLabel ) tokenFactory. makeToken(txt, Math . toIntExact( yychar) , yylength());
493
493
word. set(CoreAnnotations . OriginalTextAnnotation . class, originalText);
494
494
word. set(CoreAnnotations . BeforeAnnotation . class, str);
495
495
prevWord. set(CoreAnnotations . AfterAnnotation . class, str);
496
496
prevWord = word;
497
497
return word;
498
498
} else {
499
- Object word = tokenFactory. makeToken(txt, yychar, yylength());
499
+ Object word = tokenFactory. makeToken(txt, Math . toIntExact( yychar) , yylength());
500
500
if (word instanceof CoreLabel ) {
501
501
prevWord = (CoreLabel ) word;
502
502
}
Original file line number Diff line number Diff line change @@ -53,8 +53,8 @@ TEXT = [^ \t\u1680\u2000-\u2006\u2008-\u200A\u205F\u3000\r\n\u0085\u2028\u2029\u
53
53
54
54
%%
55
55
56
- {CR} { return tokenFactory. makeToken(NEWLINE , yychar, yylength()); }
57
- {OTHERSEP} { return tokenFactory. makeToken(NEWLINE , yychar, yylength()); }
56
+ {CR} { return tokenFactory. makeToken(NEWLINE , Math . toIntExact( yychar) , yylength()); }
57
+ {OTHERSEP} { return tokenFactory. makeToken(NEWLINE , Math . toIntExact( yychar) , yylength()); }
58
58
{SPACES} { }
59
- {TEXT} { return tokenFactory. makeToken(yytext(), yychar, yylength()); }
59
+ {TEXT} { return tokenFactory. makeToken(yytext(), Math . toIntExact( yychar) , yylength()); }
60
60
<<EOF>> { return null ; }
You can’t perform that action at this time.
0 commit comments