Merge pull request #512 from ufal/csv_limit

kocmitom · web-flow · commit 6a14f0625d4b · 2017-08-08T14:30:12.000+02:00
Avoiding csv limit
diff --git a/neuralmonkey/readers/plain_text_reader.py b/neuralmonkey/readers/plain_text_reader.py
@@ -2,6 +2,7 @@
 import gzip
 import csv
 import io
+import sys
 
 from neuralmonkey.logging import warn
 
@@ -10,6 +11,8 @@
 PlainTextFileReader = Callable[[List[str]], Iterable[List[str]]]
 # pylint: enable=invalid-name
 
+csv.field_size_limit(sys.maxsize)
+
 
 def string_reader(
         encoding: str = "utf-8") -> Callable[[List[str]], Iterable[str]]:
