Merge branch 'release-1.0.1'

Author: tmylk

CHANGELOG.md

@@ -12,6 +12,7 @@ New features:
 * Add Author-topic modeling (@olavurmortensen,[#893](https://github.com/RaRe-Technologies/gensim/pull/893))
 * Add FastText word embedding wrapper (@Jayantj,[#847](https://github.com/RaRe-Technologies/gensim/pull/847))
 * Add WordRank word embedding  wrapper (@parulsethi,[#1066](https://github.com/RaRe-Technologies/gensim/pull/1066), [#1125](https://github.com/RaRe-Technologies/gensim/pull/1125))
+* Add VarEmbed word embedding wrapper (@anmol01gulati,  [#1067](https://github.com/RaRe-Technologies/gensim/pull/1067)))
 * Add sklearn wrapper for LDAModel (@AadityaJ,[#932](https://github.com/RaRe-Technologies/gensim/pull/932))
 
 Deprecated features:
@@ -49,6 +50,7 @@ Tutorial and doc improvements:
 * Fix typos in Author-topic tutorial (@Fil,[#1102](https://github.com/RaRe-Technologies/gensim/pull/1102))
 * Address benchmark inconsistencies in Annoy tutorial (@droudy,[#1113](https://github.com/RaRe-Technologies/gensim/pull/1113))
 * Add note about Annoy speed depending on numpy BLAS setup in annoytutorial.ipynb (@greninja,[#1137](https://github.com/RaRe-Technologies/gensim/pull/1137)) 
+* Fix dependencies description on doc2vec-IMDB notebook (@luizcavalcanti, [#1132](https://github.com/RaRe-Technologies/gensim/pull/1132)) 
 * Add documentation for WikiCorpus metadata. (@kirit93, [#1163](https://github.com/RaRe-Technologies/gensim/pull/1163)) 
 
 

docs/src/conf.py

@@ -54,7 +54,7 @@
 # The short X.Y version.
 version = '1.0'
 # The full version, including alpha/beta/rc tags.
-release = '1.0.0'
+release = '1.0.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

gensim/models/wrappers/fasttext.py

@@ -277,14 +277,13 @@ def load_dict(self, file_handle):
         assert len(self.wv.vocab) == vocab_size, 'mismatch between vocab sizes'
         ntokens, = self.struct_unpack(file_handle, '@q')
         for i in range(nwords):
-            word = ''
-            char, = self.struct_unpack(file_handle, '@c')
-            char = char.decode()
+            word_bytes = b''
+            char_byte = file_handle.read(1)
             # Read vocab word
-            while char != '\x00':
-                word += char 
-                char, = self.struct_unpack(file_handle, '@c')
-                char = char.decode()
+            while char_byte != b'\x00':
+                word_bytes += char_byte
+                char_byte = file_handle.read(1)
+            word = word_bytes.decode('utf8')
             count, _ = self.struct_unpack(file_handle, '@ib')
             _ = self.struct_unpack(file_handle, '@i')
             assert self.wv.vocab[word].index == i, 'mismatch between gensim word index and fastText word index'

gensim/test/test_data/non_ascii_fasttext.bin

@@ -0,0 +1,172 @@
+171 2
+ji -1.5308 2.0551 
+který -0.99211 1.4997 
+jen -1.1228 1.3667 
+podle -1.1469 1.4473 
+zde -1.0191 1.4011 
+už -0.91921 1.3531 
+být -1.0086 1.4582 
+více -1.1058 1.3376 
+bude -1.2032 1.7383 
+již -1.3136 1.4792 
+než -1.0664 1.6635 
+vás -1.1113 1.5703 
+by -1.1698 1.966 
+které -1.1295 1.6275 
+co -0.93518 1.1776 
+nebo -1.0791 1.5071 
+ten -1.1881 1.415 
+tak -1.4548 1.8457 
+má -1.0658 1.5255 
+při -1.3464 1.6107 
+od -0.79486 1.5585 
+po -1.2758 1.9186 
+tipy -0.69335 1.0799 
+ještě -0.87116 1.1618 
+až -1.2688 1.6518 
+bez -0.99627 1.423 
+také -1.141 1.4808 
+pouze -0.94181 1.4076 
+první -1.1166 1.5035 
+vaše -0.9672 1.4975 
+která -1.1102 1.5806 
+nás -1.1328 1.5253 
+nový -0.85553 1.1462 
+jsou -1.0792 1.8008 
+pokud -1.0427 1.3178 
+může -1.1269 1.419 
+strana -0.84973 1.1957 
+jeho -1.1644 1.5879 
+své -1.0546 1.6185 
+jiné -0.95046 1.2816 
+zprávy -0.88762 1.3374 
+nové -1.0588 1.619 
+není -1.0321 1.5566 
+tomu -1.0753 1.5211 
+ona -1.21 1.6992 
+ono -1.0733 1.6574 
+oni -1.1153 1.643 
+ony -1.0926 1.5244 
+my -0.92689 1.6378 
+vy -1.3708 1.8 
+jí -1.205 1.6606 
+mě -0.96436 1.4713 
+mne -1.0956 1.6333 
+jemu -1.1181 1.4661 
+on -1.0062 1.4124 
+těm -0.90732 1.2586 
+těmu -0.90621 1.4096 
+němu -1.0823 1.4396 
+němuž -1.0786 1.3892 
+jehož -1.1649 1.4418 
+jíž -1.0574 1.6338 
+jelikož -1.0449 1.3625 
+jež -1.2657 1.7032 
+jakož -1.3373 1.6112 
+načež -1.0127 1.3696 
+ze -1.1784 1.7095 
+jak -1.2097 1.5224 
+další -0.7288 0.96256 
+ale -1.1029 1.4153 
+si -1.1097 1.5884 
+se -1.2981 1.7707 
+ve -1.256 1.7985 
+to -1.6894 2.2424 
+jako -1.2333 1.5942 
+za -1.0376 1.6162 
+zpět -0.83657 1.354 
+jejich -0.97548 1.4219 
+do -0.93685 1.4001 
+pro -1.4367 1.9498 
+je -1.9446 2.5147 
+na -1.5543 2.2901 
+atd -0.98175 1.3697 
+atp -0.83266 1.1085 
+jakmile -1.0954 1.2764 
+přičemž -1.0533 1.4279 
+já -1.1496 1.4432 
+nám -1.0246 1.6043 
+jej -1.203 1.6252 
+zda -0.93651 1.2363 
+proč -0.90395 1.3144 
+máte -0.99962 1.4802 
+tato -1.3248 1.5575 
+kam -0.63468 1.246 
+tohoto -0.9737 1.3422 
+kdo -0.88982 1.4152 
+kteří -0.92973 1.4696 
+mi -1.343 1.7217 
+tyto -0.99375 1.3067 
+tom -1.1636 1.608 
+tomuto -1.0103 1.3488 
+mít -1.1538 1.6326 
+nic -0.76497 1.0685 
+proto -1.1781 1.6367 
+kterou -1.0561 1.563 
+byla -0.9338 1.7033 
+toho -1.1263 1.5702 
+protože -1.1777 1.4984 
+asi -1.0555 1.4401 
+budeš -0.98208 1.5432 
+s -1.3733 1.6447 
+k -1.0223 1.6019 
+o -1.4531 1.879 
+i -1.0985 1.2956 
+u -0.91038 1.6173 
+v -1.2536 1.5998 
+z -0.96962 1.7437 
+dnes -0.92891 1.2478 
+cz -0.84461 1.0881 
+tímto -0.98475 1.3061 
+ho -0.74774 1.4925 
+budem -1.0178 1.4333 
+byli -0.90776 1.4799 
+jseš -1.0297 1.4975 
+můj -0.891 1.2674 
+svým -1.0586 1.5377 
+ta -1.4932 2.0156 
+tomto -1.1626 1.5135 
+tohle -1.2215 1.6529 
+tuto -1.0516 1.3583 
+neg -0.94527 1.5529 
+pod -1.0601 1.578 
+téma -0.93273 1.3456 
+mezi -0.96807 1.3465 
+přes -1.1927 1.5099 
+ty -1.3733 1.7374 
+pak -1.0392 1.5592 
+vám -0.89801 1.3586 
+ani -1.2113 1.5634 
+když -1.0124 1.5112 
+však -0.75634 1.1299 
+či -0.79489 1.2817 
+jsem -1.0435 1.4903 
+tento -1.0861 1.5053 
+článku -0.93302 1.3758 
+články -0.98897 1.4387 
+aby -1.0874 1.6114 
+jsme -1.0547 1.6846 
+před -1.0538 1.5186 
+pta -1.062 1.6063 
+a -1.3116 2.0391 
+aj -1.1578 1.5193 
+naši -1.2075 1.3714 
+napište -1.0436 1.4646 
+re -1.3115 1.5453 
+což -1.1731 1.3545 
+tím -1.0296 1.5885 
+takže -1.1014 1.3574 
+svých -0.82606 1.1187 
+její -1.1029 1.3696 
+svými -1.1052 1.4953 
+jste -1.1003 1.7465 
+byl -0.89449 1.4131 
+tu -1.1255 1.5505 
+tedy -1.1693 1.6446 
+teto -1.2134 1.546 
+bylo -0.86091 1.3805 
+kde -1.3468 1.7507 
+ke -1.0699 1.6688 
+pravé -0.9391 1.5172 
+nad -1.3404 1.7661 
+nejsou -0.85023 1.5033 

gensim/test/test_data/non_ascii_fasttext.vec

@@ -120,6 +120,14 @@ def testLoadFastTextFormat(self):
         self.assertEqual(self.test_model.wv.syn0_all.shape, (self.test_model.num_ngram_vectors, model_size))
         self.model_sanity(model)
 
+    def testLoadModelWithNonAsciiVocab(self):
+        model = fasttext.FastText.load_fasttext_format(datapath('non_ascii_fasttext'))
+        self.assertTrue(u'který' in model)
+        try:
+            vector = model[u'který']
+        except UnicodeDecodeError:
+            self.fail('Unable to access vector for non-ascii word')
+
     def testNSimilarity(self):
         """Test n_similarity for in-vocab and out-of-vocab words"""
         # In vocab, sanity check

gensim/test/test_fasttext_wrapper.py

@@ -228,7 +228,7 @@ def finalize_options(self):
 
 setup(
     name='gensim',
-    version='1.0.0',
+    version='1.0.1',
     description='Python framework for fast Vector Space Modelling',
     long_description=LONG_DESCRIPTION,