explicitly specify read buffer size when doing COPY

jqnatividad · jqnatividad · commit 814ac78ec1ae · 2024-01-23T11:43:02.000-05:00
diff --git a/datapusher/jobs.py b/datapusher/jobs.py
@@ -1441,7 +1441,7 @@ def _push_to_datastore(task_id, input, dry_run=False, temp_dir=None):
             sql.Identifier(resource_id),
             column_names,
         )
-        with open(tmp, "rb") as f:
+        with open(tmp, "rb", 8192) as f:
             try:
                 cur.copy_expert(copy_sql, f)
             except psycopg2.Error as e:

Original file line number	Diff line number	Diff line change
`@@ -1441,7 +1441,7 @@ def _push_to_datastore(task_id, input, dry_run=False, temp_dir=None):`
`1441`	`1441`	`sql.Identifier(resource_id),`
`1442`	`1442`	`column_names,`
`1443`	`1443`	`)`
`1444`		`- with open(tmp, "rb") as f:`
	`1444`	`+ with open(tmp, "rb", 8192) as f:`
`1445`	`1445`	`try:`
`1446`	`1446`	`cur.copy_expert(copy_sql, f)`
`1447`	`1447`	`except psycopg2.Error as e:`