Merge pull request #128 from dathere/syc_buffer_with_copyexpert

sync read buffer with buffer size of copyexpert

Author: jqnatividad

datapusher/dot-env.template

@@ -14,8 +14,8 @@ WRITE_ENGINE_URL = 'postgresql://datapusher:YOURPASSWORD@localhost/datastore_def
 SQLALCHEMY_DATABASE_URI = 'postgresql://datapusher_jobs:YOURPASSWORD@localhost/datapusher_jobs'
 
 # READ BUFFER SIZE IN BYTES WHEN READING CSV FILE WHEN USING POSTGRES COPY
-# default 1mb = 1048576
-COPY_READBUFFER_SIZE = 1048576
+# default 64k = 65536
+COPY_READBUFFER_SIZE = 65536
 
 # =============== DOWNLOAD SETTINGS ==============
 # 25mb, this is ignored if either PREVIEW_ROWS > 0

datapusher/jobs.py

@@ -1445,7 +1445,7 @@ def _push_to_datastore(task_id, input, dry_run=False, temp_dir=None):
         # specify a 1MB buffer size for COPY read from disk
         with open(tmp, "rb", copy_readbuffer_size) as f:
             try:
-                cur.copy_expert(copy_sql, f)
+                cur.copy_expert(copy_sql, f, size=copy_readbuffer_size)
             except psycopg2.Error as e:
                 raise util.JobError("Postgres COPY failed: {}".format(e))
             else: