Merge branch 'master' of https://github.com/xpqiu/fnlp.git

Author: xpqiu

fnlp-app/src/main/java/org/fnlp/app/lucene/SentenceTokenizer.java

@@ -1,22 +1,22 @@
-/**
-*  This file is part of FNLP (formerly FudanNLP).
-*  
-*  FNLP is free software: you can redistribute it and/or modify
-*  it under the terms of the GNU Lesser General Public License as published by
-*  the Free Software Foundation, either version 3 of the License, or
-*  (at your option) any later version.
-*  
-*  FNLP is distributed in the hope that it will be useful,
-*  but WITHOUT ANY WARRANTY; without even the implied warranty of
-*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-*  GNU Lesser General Public License for more details.
-*  
-*  You should have received a copy of the GNU General Public License
-*  along with FudanNLP.  If not, see <http://www.gnu.org/licenses/>.
-*  
-*  Copyright 2009-2014 www.fnlp.org. All rights reserved. 
-*/
-
+/**
+*  This file is part of FNLP (formerly FudanNLP).
+*  
+*  FNLP is free software: you can redistribute it and/or modify
+*  it under the terms of the GNU Lesser General Public License as published by
+*  the Free Software Foundation, either version 3 of the License, or
+*  (at your option) any later version.
+*  
+*  FNLP is distributed in the hope that it will be useful,
+*  but WITHOUT ANY WARRANTY; without even the implied warranty of
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*  GNU Lesser General Public License for more details.
+*  
+*  You should have received a copy of the GNU General Public License
+*  along with FudanNLP.  If not, see <http://www.gnu.org/licenses/>.
+*  
+*  Copyright 2009-2014 www.fnlp.org. All rights reserved. 
+*/
+
 package org.fnlp.app.lucene;
 
 import java.io.IOException;
@@ -38,7 +38,7 @@ public final class SentenceTokenizer extends Tokenizer {
   /**
    * Space-like characters that need to be skipped: such as space, tab, newline, carriage return.
    */
-  public static final String SPACES = " 　\t\r\n";
+  public static final String SPACES = " \t\r\n";
 
   private final StringBuilder buffer = new StringBuilder();
 

fnlp-app/src/main/java/org/fnlp/app/lucene/WordTokenFilter.java

@@ -1,22 +1,22 @@
-/**
-*  This file is part of FNLP (formerly FudanNLP).
-*  
-*  FNLP is free software: you can redistribute it and/or modify
-*  it under the terms of the GNU Lesser General Public License as published by
-*  the Free Software Foundation, either version 3 of the License, or
-*  (at your option) any later version.
-*  
-*  FNLP is distributed in the hope that it will be useful,
-*  but WITHOUT ANY WARRANTY; without even the implied warranty of
-*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-*  GNU Lesser General Public License for more details.
-*  
-*  You should have received a copy of the GNU General Public License
-*  along with FudanNLP.  If not, see <http://www.gnu.org/licenses/>.
-*  
-*  Copyright 2009-2014 www.fnlp.org. All rights reserved. 
-*/
-
+/**
+*  This file is part of FNLP (formerly FudanNLP).
+*  
+*  FNLP is free software: you can redistribute it and/or modify
+*  it under the terms of the GNU Lesser General Public License as published by
+*  the Free Software Foundation, either version 3 of the License, or
+*  (at your option) any later version.
+*  
+*  FNLP is distributed in the hope that it will be useful,
+*  but WITHOUT ANY WARRANTY; without even the implied warranty of
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*  GNU Lesser General Public License for more details.
+*  
+*  You should have received a copy of the GNU General Public License
+*  along with FudanNLP.  If not, see <http://www.gnu.org/licenses/>.
+*  
+*  Copyright 2009-2014 www.fnlp.org. All rights reserved. 
+*/
+
 package org.fnlp.app.lucene;
 
 import java.io.IOException;
@@ -81,8 +81,9 @@ public boolean incrementToken() throws IOException {
 				posBuffer = Arrays.asList(p);
 				tokenIter = tokenBuffer.iterator();
 				posIter = posBuffer.iterator();
-				idx = 0;
-				/* 
+//				idx = 0;
+				idx = tokStart;
+				/*
 				 * it should not be possible to have a sentence with 0 words, check just in case.
 				 * returning EOS isn't the best either, but its the behavior of the original code.
 				 */
@@ -103,7 +104,7 @@ public boolean incrementToken() throws IOException {
 		if (hasIllegalOffsets) {
 			offsetAtt.setOffset(tokStart, tokEnd);
 		} else {
-			offsetAtt.setOffset(idx, end-1);
+			offsetAtt.setOffset(idx, end);
 		}
 		idx = end;
 		typeAtt.setType("word");