normalize via Unicode NFC

Author: bertsky

Makefile

@@ -5,7 +5,7 @@
 all: all-tess all-hunspell
 
 install:
-	sudo apt-get install sqlite3 wget
+	sudo apt-get install sqlite3 wget icu-devtools
 	sudo add-apt-repository -u -y ppa:alex-p/tesseract-ocr
 	sudo apt-get update
 	sudo apt-get install tesseract-ocr

sql2wordlist.sh

@@ -1,4 +1,4 @@
 #!/bin/sh
-sqlite3 -list -readonly -noheader ${1:-lexdb.sqlite} <<EOF | grep -v -e '^[[:punct:]]' -e '^[[:digit:][:punct:]]*$' > ${2:-lexdb_${3:-100}.words}
+sqlite3 -list -readonly -noheader ${1:-lexdb.sqlite} <<EOF | uconv -f utf-8 -t utf-8 -x "::nfc;" | grep -v -e '^[[:punct:]]' -e '^[[:digit:][:punct:]]*$' > ${2:-lexdb_${3:-100}.words}
 select trim(u) from csv where f > ${3:-100} and p != "\$(" and p != "\$," and p != "\$." and p != "FM.xy" and p != "CARD" and p != "XY";
 EOF