fix bug

Author: xpqiu

fnlp-core/src/main/java/org/fnlp/data/reader/DocumentReader.java

@@ -40,7 +40,6 @@
  * 不同类别的文件放在不同的子文件夹下
  * 类别：子文件夹名
  * 数据：文件内所有字符
- * package edu.fudan.ml.data
  */
 public class DocumentReader extends Reader {
 
@@ -87,7 +86,8 @@ private void nextDocument() {
 		} catch (IOException e) {
 			e.printStackTrace();
 		}
-		cur = new Instance(buff.toString(), f.getPath());
+	String label = f.getParentFile().getName();
+		cur = new Instance(buff.toString(), label);
 		buff = null;
 	}
 }

fnlp-core/src/main/java/org/fnlp/data/reader/package-info.java

@@ -0,0 +1,23 @@
+/**
+*  This file is part of FNLP (formerly FudanNLP).
+*  
+*  FNLP is free software: you can redistribute it and/or modify
+*  it under the terms of the GNU Lesser General Public License as published by
+*  the Free Software Foundation, either version 3 of the License, or
+*  (at your option) any later version.
+*  
+*  FNLP is distributed in the hope that it will be useful,
+*  but WITHOUT ANY WARRANTY; without even the implied warranty of
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*  GNU Lesser General Public License for more details.
+*  
+*  You should have received a copy of the GNU General Public License
+*  along with FudanNLP.  If not, see <http://www.gnu.org/licenses/>.
+*  
+*  Copyright 2009-2014 www.fnlp.org. All rights reserved. 
+*/
+
+/**
+ * 数据读取包：处理不同类型格式的数据。
+ */
+package org.fnlp.data.reader;

fnlp-core/src/main/java/org/fnlp/data/reader/package.html

@@ -21,7 +21,10 @@
 
 import gnu.trove.iterator.TIntIterator;
 import gnu.trove.list.array.TIntArrayList;
-
+/**
+ * 0/1取值的稀疏向量
+ * @author xpqiu
+ */
 public class BinarySparseVector implements ISparseVector {
 
 
@@ -42,7 +45,7 @@ public float dotProduct(float[] vector) {
 		float sum = 0f;
 		while(it.hasNext()){
 			int i = it.next();
-			if(i<0)
+			if(i<0||i>=vector.length)
 				continue;
 			sum += vector[i];
 		}

fnlp-core/src/main/java/org/fnlp/ml/types/sv/BinarySparseVector.java

@@ -56,7 +56,7 @@ public static TreeSet<String> print(String s) {
 	}
 
 	public static String ngram(String ss,int n) {
-		List l = NGram.ngram(ss, new int[]{n});
+		List l = NGram.ngramOnCharacter2List(ss, new int[]{n});
 		StringBuilder sb = new StringBuilder();
 		for(int i=0;i<l.size();i++){
 			sb.append(l.get(i));
@@ -66,7 +66,7 @@ public static String ngram(String ss,int n) {
 	}
 
 	public static Set<String> ngramSet(String ss,int n) {
-		return NGram.ngramSet(ss, new int[]{n});		
+		return NGram.ngramOnCharacter2Set(ss, new int[]{n});		
 	}
 
 	public static String feature(String s, Type t) {

fnlp-core/src/main/java/org/fnlp/nlp/duplicate/FingerPrint.java

@@ -26,7 +26,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
-
+
 import org.fnlp.ml.types.Instance;
 import org.fnlp.util.exception.UnsupportedDataTypeException;
 
@@ -40,7 +40,16 @@
 public class NGram extends Pipe{
 
     private static final long serialVersionUID = -2329969202592736092L;
-    int[] gramSizes = null;
+    int[] gramSizes = null;
+    /**
+     * 用基于字而不是词的Ngram
+     */
+	private boolean useCharacter = false;
+    
+    public NGram(int[] sizes,boolean useCharacter) {
+        this.gramSizes = sizes;
+        this.useCharacter = useCharacter;
+    }
 
     public NGram(int[] sizes) {
         this.gramSizes = sizes;
@@ -50,8 +59,13 @@ public NGram(int[] sizes) {
     public void addThruPipe(Instance inst) throws UnsupportedDataTypeException {
         Object data = inst.getData();
         ArrayList<String> list = null;
-        if (data instanceof String) {
-            list = ngram((String) data,gramSizes);
+        if (data instanceof String) {
+        	if(useCharacter)
+        		list = ngramOnCharacter2List((String) data,gramSizes);
+        	else{ 
+        		 list = ngram((String) data,gramSizes);
+        	}
+        		
         }else if (data instanceof List) {
             list = ngram((List) data,gramSizes);
         }else if(data instanceof String[]){
@@ -63,7 +77,12 @@ public void addThruPipe(Instance inst) throws UnsupportedDataTypeException {
     }
 
 
-    /**
+    private ArrayList<String> ngram(String str, int[] gramSizes) {
+    	String[] toks = str.split("\\s+");
+    	return ngram(toks,gramSizes);
+	}
+
+	/**
      * 抽取ngram
      * @param strs
      * @param grams
@@ -123,10 +142,10 @@ private ArrayList<String> ngram(List tokens, int[] gramSizes2) {
      * @param gramSizes
      * @return ngram字符串数组
      */
-    public static ArrayList<String> ngram(String data,int[] gramSizes) {
+    public static ArrayList<String> ngramOnCharacter2List(String data,int[] gramSizes) {
         // 提取ngram
         ArrayList<String> list = new ArrayList<String>();
-        ngram(data, gramSizes, list);
+        ngramOnCharacter(data, gramSizes, list);
         return list;
     }
 
@@ -136,10 +155,10 @@ public static ArrayList<String> ngram(String data,int[] gramSizes) {
      * @param gramSizes
      * @return ngram字符串集合
      */
-    public static Set<String> ngramSet(String data,int[] gramSizes) {
+    public static Set<String> ngramOnCharacter2Set(String data,int[] gramSizes) {
         // 提取ngram
         Set<String> list = new HashSet<String>();
-        ngram(data, gramSizes, list);
+        ngramOnCharacter(data, gramSizes, list);
         return list;
     }
 
@@ -149,7 +168,7 @@ public static Set<String> ngramSet(String data,int[] gramSizes) {
      * @param gramSizes
      * @param list
      */
-    private static void ngram(String data, int[] gramSizes, Collection<String> list) {
+    private static void ngramOnCharacter(String data, int[] gramSizes, Collection<String> list) {
         for (int j = 0; j < gramSizes.length; j++) {
             int len = gramSizes[j];
             if (len <= 0 || len > data.length())

fnlp-core/src/main/java/org/fnlp/nlp/pipe/NGram.java

@@ -66,17 +66,18 @@ public static void main(String[] args) throws Exception {
 		Pipe targetpp = new Target2Label(af.DefaultLabelAlphabet());		
 
 		//建立pipe组合
-		SeriesPipes pp = new SeriesPipes(new Pipe[]{ngrampp,targetpp,indexpp});
+		SeriesPipes pp = new SeriesPipes(new Pipe[]{targetpp,ngrampp,indexpp});
 
 		InstanceSet trainset = new InstanceSet(pp,af);
+		
 		InstanceSet testset = new InstanceSet(pp,af);
 
 		//用不同的Reader读取相应格式的文件
 		Reader reader = new DocumentReader(trainDataPath);
 
 		//读入数据，并进行数据处理
 		trainset.loadThruStagePipes(reader);
-		
+//		af.setStopIncrement(true);
 		reader = new DocumentReader(testDataPath);
 
 		testset.loadThruStagePipes(reader);