Merge pull request #17 from tesseract2048/master

Optimize MurmurHash, benchmarked 33% faster

Adding the serialVersionUID for incompatible problem.

Author: xpqiu

fnlp-core/src/main/java/org/fnlp/nlp/cn/tag/CWSTagger.java

@@ -229,8 +229,8 @@ public static void main(String[] args) throws Exception {
 			HelpFormatter f = new HelpFormatter();
 			f.printHelp(
 					"SEG:\n"
-							+ "java edu.fudan.nlp.tag.CWSTagger -f model_file input_file output_file;\n"
-							+ "java edu.fudan.nlp.tag.CWSTagger -s model_file string_to_segement",
+							+ "java org.fnlp.tag.CWSTagger -f model_file input_file output_file;\n"
+							+ "java org.fnlp.tag.CWSTagger -s model_file string_to_segement",
 							opt);
 			return;
 		}

fnlp-core/src/main/java/org/fnlp/util/MyStrings.java

@@ -1,21 +1,21 @@
 /**
-*  This file is part of FNLP (formerly FudanNLP).
-*  
-*  FNLP is free software: you can redistribute it and/or modify
-*  it under the terms of the GNU Lesser General Public License as published by
-*  the Free Software Foundation, either version 3 of the License, or
-*  (at your option) any later version.
-*  
-*  FNLP is distributed in the hope that it will be useful,
-*  but WITHOUT ANY WARRANTY; without even the implied warranty of
-*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-*  GNU Lesser General Public License for more details.
-*  
-*  You should have received a copy of the GNU General Public License
-*  along with FudanNLP.  If not, see <http://www.gnu.org/licenses/>.
-*  
-*  Copyright 2009-2014 www.fnlp.org. All rights reserved. 
-*/
+ *  This file is part of FNLP (formerly FudanNLP).
+ *  
+ *  FNLP is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Lesser General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  FNLP is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Lesser General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with FudanNLP.  If not, see <http://www.gnu.org/licenses/>.
+ *  
+ *  Copyright 2009-2014 www.fnlp.org. All rights reserved. 
+ */
 
 package org.fnlp.util;
 
@@ -116,9 +116,9 @@ public static String toString(float[] s, String delim) {
 				sb.append(delim);
 		}
 		return sb.toString();
-		
+
 	}
-	
+
 	/**
 	 * 字符在字符串中出现的次数
 	 * 
@@ -155,8 +155,28 @@ public static String toString(float[][] s) {
 				sb.append("\n");
 		}
 		return sb.toString();
+	}
+	/**
+	 * 统计文本中某个字符串出现的次数
+	 * @param str
+	 * @param target
+	 * @return
+	 * 下午5:33:24
+	 */
+	public static int count(String str,String target){
+		int count=0; 
+		int index=0; 
+		while(true){ 
+			index=str.indexOf(target,index+1); 
+			if(index>0){ 
+				count++; 
+			}else{ 
+				break; 
+			} 
+		} 
+		return count; 
 	}
 
-	
+
 
 }

fnlp-core/src/main/java/org/fnlp/util/Options.java

@@ -0,0 +1,32 @@
+/**
+ * 
+ */
+package org.fnlp.util;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+
+/**
+ * @author Xipeng Qiu  E-mail: xpqiu@fudan.edu.cn
+ * @version 创建时间：2015年1月6日 下午4:39:23
+ * @since fnlp 2.1
+ */
+public class Options {
+	HashMap<String,String> options = new HashMap<String,String>();
+	ArrayList<String> rootArgs = new ArrayList<String>();
+	
+	
+	public void parsing(String[] args) {
+		
+		for (int i=0; i<args.length; ++i) {
+			if (args[i].charAt(0) != '-') {
+				rootArgs.add(args[i]);
+			} else if (i+1 < args.length) {
+				options.put(args[i], args[++i]);
+			} else {
+				throw new IllegalArgumentException();
+			}
+		}
+	}
+
+}

fnlp-core/src/main/java/org/fnlp/util/hash/MurmurHash.java

@@ -19,10 +19,7 @@
 
 package org.fnlp.util.hash;
 
-import java.io.UnsupportedEncodingException;
-
-
-/** 
+/**
  *
  * murmur hash 2.0.
  * 
@@ -40,12 +37,27 @@
  *
  */
 public final class MurmurHash extends AbstractHashCode {
+	
+	private static final long serialVersionUID = 4342869264396184799L;
 
 	// all methods static; private constructor. 
 	public MurmurHash() {}
 
+    protected byte[] toBytesWithoutEncoding(String str) {
+        int len = str.length();
+        int pos = 0;
+        byte[] buf = new byte[len << 1];
+        for (int i = 0; i < len; i++) {
+            char c = str.charAt(i);
+            buf[pos++] = (byte) (c & 0xFF);
+            buf[pos++] = (byte) (c >> 8);
+        }
+        return buf;
+    }
+
 	public int hashcode(String str) {
-		return hash32(str);
+        byte[] bytes = toBytesWithoutEncoding(str);
+        return hash32(bytes, bytes.length);
 	}
 
 	/** 
@@ -106,31 +118,6 @@ public int hash32( final byte[] data, int length) {
 	}
 
 
-	/** 
-	 * Generates 32 bit hash from a string.
-	 * 
-	 * @param text string to hash
-	 * @return 32 bit hash of the given string
-	 */
-	public int hash32( final String text) {
-		final byte[] bytes = text.getBytes(); 
-		return hash32( bytes, bytes.length);
-	}
-
-
-	/** 
-	 * Generates 32 bit hash from a substring.
-	 * 
-	 * @param text string to hash
-	 * @param from starting index
-	 * @param length length of the substring to hash
-	 * @return 32 bit hash of the given string
-	 */
-	public int hash32( final String text, int from, int length) {
-		return hash32( text.substring( from, from+length));
-	}
-
-
 	/** 
 	 * Generates 64 bit hash from byte array of the given length and seed.
 	 * 
@@ -192,44 +179,4 @@ public long hash64( final byte[] data, int length) {
 		return hash64( data, length, 0xe17a1465);
 	}
 
-
-	/** 
-	 * Generates 64 bit hash from a string.
-	 * 
-	 * @param text string to hash
-	 * @return 64 bit hash of the given string
-	 */
-	public long hash64( final String text) {
-
-		final byte[] bytes;
-		try {
-			bytes = text.getBytes("utf-8");
-
-			return hash32( bytes, bytes.length);
-		} catch (UnsupportedEncodingException e) {
-			throw new RuntimeException("utf-8 encoding should be available on system.", e);
-		}
-	}
-
-
-
-	/** 
-	 * Generates 64 bit hash from a substring.
-	 * 
-	 * @param text string to hash
-	 * @param from starting index
-	 * @param length length of the substring to hash
-	 * @return 64 bit hash of the given array
-	 */
-	public long hash64( final String text, int from, int length) {
-
-		final byte[] bytes;
-		try {
-			bytes = text.getBytes("utf-8");
-
-			return hash64( bytes, bytes.length);
-		} catch (UnsupportedEncodingException e) {
-			throw new RuntimeException("utf-8 encoding should be available on system.", e);
-		}
-	}
-}
+}

fnlp-core/src/test/java/org/fnlp/util/hash/MurmurHashTest.java

@@ -0,0 +1,54 @@
+/**
+ * 
+ */
+package org.fnlp.util.hash;
+
+import org.fnlp.nlp.cn.tag.CWSTagger;
+import org.fnlp.util.exception.LoadModelException;
+
+/**
+ * @author Xipeng Qiu  E-mail: xpqiu@fudan.edu.cn
+ * @version 创建时间：2015年1月8日 下午3:14:28
+ */
+public class MurmurHashTest {
+
+	private CWSTagger cws;
+
+	public MurmurHashTest() {
+		try {
+			long t = System.currentTimeMillis();
+			cws = new CWSTagger("../models/seg.m");
+			long elapsed = System.currentTimeMillis() - t;
+			System.out.println("Fnlp loaded in " + elapsed + " ms.");
+		} catch (LoadModelException e) {
+			throw new RuntimeException("Failed to load fnlp", e);
+		}
+	}
+	
+
+	protected String[] getCws(String text) {
+		return cws.tag2Array(text);
+	}
+
+	protected void benchmark() {
+		long t = System.currentTimeMillis();
+		String input = "12月1日，长江经济带海关区域通关一体化改革实现流域全覆盖，南昌、武汉、长沙、成都、重庆、贵阳、昆明等7个海关加入改革。当天，流域12个关区的海关特殊监管区域也纳入区域通关一体化，长江全流域真正实现了“12关如1关”。这标志着京津冀、长江经济带、广东地区三大区域通关一体化改革全面实施";
+		for (int i = 0; i < 5000; i++) {
+			getCws(input);
+		}
+		long elapsed = System.currentTimeMillis() - t;
+		System.out.println("Benchmarked " + elapsed + " ms.");
+	}
+
+	public void run() {
+		// warm up the code, and perform benchmark
+		for (int k = 0; k < 10; k ++) {
+			benchmark();
+		}
+	}
+
+	public static void main(String[] args) throws InterruptedException {
+		new MurmurHashTest().run();
+	}
+}
+