更新训练程序

Author: xpqiu

README.md

@@ -43,7 +43,11 @@ You can also use the following site to check the partial functionality.
 
 [FNLP入门教程](https://github.com/xpqiu/fnlp/wiki)
 
-大的模型文件和代码分离。相应的模型文件可以从release页面下载。
+除了源码文件，还需要下载FNLP模型文件。由于模型文件较大，不便于存放在源码库之中，请至[Release](https://github.com/xpqiu/fnlp/releases)页面下载，并将模型文件放在“models”目录。
+
+* seg.m 分词模型
+* pos.m 词性标注模型
+* dep.m 依存句法分析模型
 
 欢迎大家提供非Java语言的接口。
 

fnlp-core/src/main/java/org/fnlp/ml/classifier/bayes/BayesClassifier.java

@@ -11,24 +11,20 @@
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import java.io.Serializable;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
 
 import org.fnlp.ml.classifier.AbstractClassifier;
-import org.fnlp.ml.classifier.LabelParser.Type;
-import org.fnlp.ml.classifier.linear.Linear;
 import org.fnlp.ml.classifier.LabelParser;
+import org.fnlp.ml.classifier.LabelParser.Type;
 import org.fnlp.ml.classifier.Predict;
-import org.fnlp.ml.classifier.TPredict;
 import org.fnlp.ml.feature.FeatureSelect;
 import org.fnlp.ml.types.Instance;
 import org.fnlp.ml.types.alphabet.AlphabetFactory;
 import org.fnlp.ml.types.sv.HashSparseVector;
 import org.fnlp.nlp.pipe.Pipe;
 import org.fnlp.util.exception.LoadModelException;
-import org.junit.Ignore;
 /**
  * 朴素贝叶斯分类器
  * @author sywu

fnlp-core/src/main/java/org/fnlp/nlp/corpus/StopWords.java

@@ -135,15 +135,35 @@ public List<String> phraseDel(String[] words){
 		return list;
 	}
 
-	Pattern noise = Pattern.compile(".*["+CharSets.allRegexPunc+"\\d]+.*");
-	
-	public boolean isStopWord(String word) {
-		if (word.length() == 1 || word.length()>4)
-			return true;
-
-		if (noise.matcher(word).matches())
-			return true;
-
+	Pattern noise = Pattern.compile(".*["+CharSets.allRegexPunc+"\\d]+.*");
+	
+	/**
+	 * 判断是否为停用词
+	 * @param word
+	 * @param minLen 最小长度
+	 * @param maxLen 最大长度
+	 * @return
+	 */
+	public boolean isStopWord(String word,int minLen, int maxLen) {
+		if (word.length() < minLen || word.length()>maxLen)
+			return true;
+
+		if (noise.matcher(word).matches())
+			return true;
+
+		if (sWord.contains(word))
+			return true;
+
+		return false;
+	}
+	
+	/**
+	 * 判断是否为停用词
+	 * @param word
+	 * @return
+	 */
+	public boolean isStopWord(String word) {		
+		
 		if (sWord.contains(word))
 			return true;
 

fnlp-core/src/main/java/org/fnlp/nlp/similarity/train/WordCluster.java

@@ -543,7 +543,7 @@ public Cluster startClustering() {
 					return null;
 			}
 			try {
-				saveTxt("res-"+round);
+				saveTxt("../tmp/res-"+round);
 			} catch (Exception e) {
 				// TODO Auto-generated catch block
 				e.printStackTrace();

fnlp-core/src/test/java/org/fnlp/core/AppTest.java

@@ -0,0 +1,36 @@
+package org.fnlp.nlp.corpus;
+
+import static org.junit.Assert.*;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class StopWordsTest {
+
+	@BeforeClass
+	public static void setUpBeforeClass() throws Exception {
+	}
+
+	@AfterClass
+	public static void tearDownAfterClass() throws Exception {
+	}
+
+	@Test
+	public void testIsStopWordStringIntInt() {
+		StopWords sw = new StopWords();
+		sw.read("../models/stopwords/StopWords.txt");
+		assertTrue(!sw.isStopWord("现在我",2,4));
+		assertTrue(sw.isStopWord("我0",2,4));
+		assertTrue(sw.isStopWord("我#",2,4));	
+		assertTrue(sw.isStopWord(" ",2,4));	
+	}
+
+	@Test
+	public void testIsStopWordString() {
+		StopWords sw = new StopWords();
+		sw.read("../models/stopwords/StopWords.txt");
+		assertTrue(!sw.isStopWord("现在我"));
+	}
+
+}

fnlp-core/src/test/java/org/fnlp/nlp/corpus/StopWordsTest.java

@@ -20,17 +20,5 @@ public static void setUpBeforeClass() throws Exception {
 	public static void tearDownAfterClass() throws Exception {
 	}
 
-	@Test
-	public void testCombineStringStringArray() {
-		fail("Not yet implemented");
-	}
-
-	@Test
-	public void testCombineStringFileArray() throws Exception {
-		List<File> files = MyFiles.getAllFiles("../tmp/", ".cws");
-		MyFiles.combine("../tmp/all.cws",files.toArray(new File[files.size()]));  
-		System.out.println(new Date().toString());
-		System.out.println("Done!");
-	}
 
 }

fnlp-core/src/test/java/org/fnlp/nlp/resources/StopWordsTest.java

@@ -1,10 +1,8 @@
 package org.fnlp.demo.nlp.tc;
 
-import org.fnlp.data.reader.DocumentReader;
-import org.fnlp.data.reader.FileReader;
 import org.fnlp.data.reader.Reader;
-import org.fnlp.ml.classifier.Predict;
 import org.fnlp.ml.classifier.LabelParser.Type;
+import org.fnlp.ml.classifier.Predict;
 import org.fnlp.ml.classifier.bayes.BayesClassifier;
 import org.fnlp.ml.classifier.bayes.BayesTrainer;
 import org.fnlp.ml.classifier.knn.KNNClassifier;

fnlp-core/src/test/java/org/fnlp/util/MyFilesTest.java

@@ -57,8 +57,8 @@ public class RLSeg {
 	BufferedWriter bwNew;
 	BufferedWriter bwNo;
 	float prop = 0.5f;
-	private String newdictfile = "./tmp/dict-new.txt";
-	private String nodictfile = "./tmp/dict-no.txt";
+	private String newdictfile = "../tmp/dict-new.txt";
+	private String nodictfile = "../tmp/dict-no.txt";
 
 	public RLSeg(CWSTagger tag, String string) throws IOException{
 		this.tag = tag;
@@ -88,7 +88,7 @@ public static void main(String[] args) throws Exception {
 		tag.setDictionary(tempdict);
 
 		System.out.println("\n处理文件：");
-		String s4 = tag.tagFile("./example-data/data-tag.txt");
+		String s4 = tag.tagFile("../example-data/data-tag.txt");
 		System.out.println(s4);
 		String[] toks = s4.split("\\s+");
 		int newset = rlseg.update(toks);
@@ -97,7 +97,7 @@ public static void main(String[] args) throws Exception {
 		tag.setDictionary(tempdict);
 
 		System.out.println("\n处理文件：");
-		String s = tag.tagFile("./example-data/data-tag.txt");
+		String s = tag.tagFile("../example-data/data-tag.txt");
 		System.out.println(s);