Skip to content

Commit a06cb38

Browse files
techousetechouse
committed
✨ Add option to update duplicate records
1 parent f83293d commit a06cb38

File tree

6 files changed

+181
-46
lines changed

6 files changed

+181
-46
lines changed

‎README.md

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,11 +53,17 @@ Options:
5353
-h, --mysql-host TEXT MySQL host. Defaults to localhost.
5454
-P, --mysql-port INTEGER MySQL port. Defaults to 3306.
5555
-S, --skip-ssl Disable MySQL connection encryption.
56+
-i, --mysql-insert-method [UPDATE|IGNORE|DEFAULT]
57+
MySQL insert method. DEFAULT will throw
58+
errors when encountering duplicate records;
59+
UPDATE will update existing rows; IGNORE
60+
will ignore insert errors. Defaults to
61+
IGNORE.
5662
--mysql-integer-type TEXT MySQL default integer field type. Defaults
5763
to INT(11).
5864
--mysql-string-type TEXT MySQL default string field type. Defaults to
5965
VARCHAR(255).
60-
--mysql-text-type [LONGTEXT|TINYTEXT|MEDIUMTEXT|TEXT]
66+
--mysql-text-type [MEDIUMTEXT|TINYTEXT|TEXT|LONGTEXT]
6167
MySQL default text field type. Defaults to
6268
TEXT.
6369
--mysql-charset TEXT MySQL database and table character set

‎sqlite3_to_mysql/cli.py

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,11 @@
99
from . import SQLite3toMySQL
1010
from .click_utils import OptionEatAll, prompt_password
1111
from .debug_info import info
12-
from .mysql_utils import MYSQL_TEXT_COLUMN_TYPES, mysql_supported_character_sets
12+
from .mysql_utils import (
13+
MYSQL_INSERT_METHOD,
14+
MYSQL_TEXT_COLUMN_TYPES,
15+
mysql_supported_character_sets,
16+
)
1317

1418

1519
@click.command()
@@ -61,6 +65,14 @@
6165
@click.option(
6266
"-S", "--skip-ssl", is_flag=True, help="Disable MySQL connection encryption."
6367
)
68+
@click.option(
69+
"-i",
70+
"--mysql-insert-method",
71+
type=click.Choice(MYSQL_INSERT_METHOD, case_sensitive=False),
72+
default="IGNORE",
73+
help="MySQL insert method. DEFAULT will throw errors when encountering duplicate records; "
74+
"UPDATE will update existing rows; IGNORE will ignore insert errors. Defaults to IGNORE.",
75+
)
6476
@click.option(
6577
"--mysql-integer-type",
6678
default="INT(11)",
@@ -127,6 +139,7 @@ def cli(
127139
mysql_host,
128140
mysql_port,
129141
skip_ssl,
142+
mysql_insert_method,
130143
mysql_integer_type,
131144
mysql_string_type,
132145
mysql_text_type,
@@ -167,6 +180,7 @@ def cli(
167180
mysql_host=mysql_host,
168181
mysql_port=mysql_port,
169182
mysql_ssl_disabled=skip_ssl,
183+
mysql_insert_method=mysql_insert_method,
170184
mysql_integer_type=mysql_integer_type,
171185
mysql_string_type=mysql_string_type,
172186
mysql_text_type=mysql_text_type,

‎sqlite3_to_mysql/mysql_utils.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,12 @@
6969

7070
CharSet = namedtuple("CharSet", ["id", "charset", "collation"])
7171

72+
MYSQL_INSERT_METHOD = {
73+
"DEFAULT",
74+
"IGNORE",
75+
"UPDATE",
76+
}
77+
7278

7379
def mysql_supported_character_sets(charset=None):
7480
"""Get supported MySQL character sets."""

‎sqlite3_to_mysql/transporter.py

Lines changed: 45 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import sqlite3
88
from datetime import timedelta
99
from decimal import Decimal
10+
from itertools import chain
1011
from math import ceil
1112
from os.path import isfile, realpath
1213
from sys import stdout
@@ -30,6 +31,7 @@
3031
MYSQL_BLOB_COLUMN_TYPES,
3132
MYSQL_COLUMN_TYPES,
3233
MYSQL_COLUMN_TYPES_WITHOUT_DEFAULT,
34+
MYSQL_INSERT_METHOD,
3335
MYSQL_TEXT_COLUMN_TYPES,
3436
MYSQL_TEXT_COLUMN_TYPES_WITH_JSON,
3537
check_mysql_fulltext_support,
@@ -94,6 +96,12 @@ def __init__(self, **kwargs):
9496

9597
self._mysql_database = str(kwargs.get("mysql_database") or "transfer")
9698

99+
self._mysql_insert_method = str(
100+
kwargs.get("mysql_integer_type") or "IGNORE"
101+
).upper()
102+
if self._mysql_insert_method not in MYSQL_INSERT_METHOD:
103+
self._mysql_insert_method = "IGNORE"
104+
97105
self._mysql_integer_type = str(
98106
kwargs.get("mysql_integer_type") or "INT(11)"
99107
).upper()
@@ -692,15 +700,43 @@ def transfer(self):
692700
safe_identifier_length(column[0])
693701
for column in self._sqlite_cur.description
694702
]
695-
sql = """
696-
INSERT IGNORE
697-
INTO `{table}` ({fields})
698-
VALUES ({placeholders})
699-
""".format(
700-
table=safe_identifier_length(table["name"]),
701-
fields=("`{}`, " * len(columns)).rstrip(" ,").format(*columns),
702-
placeholders=("%s, " * len(columns)).rstrip(" ,"),
703-
)
703+
if self._mysql_insert_method.upper() == "UPDATE":
704+
sql = """
705+
INSERT
706+
INTO `{table}` ({fields})
707+
VALUES ({placeholders}) AS `__new__`
708+
ON DUPLICATE KEY UPDATE {field_updates}
709+
""".format(
710+
table=safe_identifier_length(table["name"]),
711+
fields=("`{}`, " * len(columns))
712+
.rstrip(" ,")
713+
.format(*columns),
714+
placeholders=("%s, " * len(columns)).rstrip(" ,"),
715+
field_updates=("`{}`=`__new__`.`{}`, " * len(columns))
716+
.rstrip(" ,")
717+
.format(
718+
*list(
719+
chain.from_iterable(
720+
(column, column) for column in columns
721+
)
722+
)
723+
),
724+
)
725+
else:
726+
sql = """
727+
INSERT {ignore}
728+
INTO `{table}` ({fields})
729+
VALUES ({placeholders})
730+
""".format(
731+
ignore="IGNORE"
732+
if self._mysql_insert_method.upper() == "IGNORE"
733+
else "",
734+
table=safe_identifier_length(table["name"]),
735+
fields=("`{}`, " * len(columns))
736+
.rstrip(" ,")
737+
.format(*columns),
738+
placeholders=("%s, " * len(columns)).rstrip(" ,"),
739+
)
704740
try:
705741
self._transfer_table_data(sql=sql, total_records=total_records)
706742
except mysql.connector.Error as err:

‎tests/func/sqlite3_to_mysql_test.py

Lines changed: 62 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -218,7 +218,33 @@ def test_log_to_file(
218218

219219
@pytest.mark.transfer
220220
@pytest.mark.parametrize(
221-
"chunk, with_rowid", [(None, False), (None, True), (10, False), (10, True)]
221+
"chunk, with_rowid, mysql_insert_method, ignore_duplicate_keys",
222+
[
223+
(None, False, "IGNORE", False),
224+
(None, False, "IGNORE", True),
225+
(None, False, "UPDATE", True),
226+
(None, False, "UPDATE", False),
227+
(None, False, "DEFAULT", True),
228+
(None, False, "DEFAULT", False),
229+
(None, True, "IGNORE", False),
230+
(None, True, "IGNORE", True),
231+
(None, True, "UPDATE", True),
232+
(None, True, "UPDATE", False),
233+
(None, True, "DEFAULT", True),
234+
(None, True, "DEFAULT", False),
235+
(10, False, "IGNORE", False),
236+
(10, False, "IGNORE", True),
237+
(10, False, "UPDATE", True),
238+
(10, False, "UPDATE", False),
239+
(10, False, "DEFAULT", True),
240+
(10, False, "DEFAULT", False),
241+
(10, True, "IGNORE", False),
242+
(10, True, "IGNORE", True),
243+
(10, True, "UPDATE", True),
244+
(10, True, "UPDATE", False),
245+
(10, True, "DEFAULT", True),
246+
(10, True, "DEFAULT", False),
247+
],
222248
)
223249
def test_transfer_transfers_all_tables_in_sqlite_file(
224250
self,
@@ -230,6 +256,8 @@ def test_transfer_transfers_all_tables_in_sqlite_file(
230256
caplog,
231257
chunk,
232258
with_rowid,
259+
mysql_insert_method,
260+
ignore_duplicate_keys,
233261
):
234262
proc = SQLite3toMySQL(
235263
sqlite_file=sqlite_database,
@@ -240,6 +268,8 @@ def test_transfer_transfers_all_tables_in_sqlite_file(
240268
mysql_database=mysql_credentials.database,
241269
chunk=chunk,
242270
with_rowid=with_rowid,
271+
mysql_insert_method=mysql_insert_method,
272+
ignore_duplicate_keys=ignore_duplicate_keys,
243273
)
244274
caplog.set_level(logging.DEBUG)
245275
proc.transfer()
@@ -385,7 +415,33 @@ def test_transfer_transfers_all_tables_in_sqlite_file(
385415

386416
@pytest.mark.transfer
387417
@pytest.mark.parametrize(
388-
"chunk, with_rowid", [(None, False), (None, True), (10, False), (10, True)]
418+
"chunk, with_rowid, mysql_insert_method, ignore_duplicate_keys",
419+
[
420+
(None, False, "IGNORE", False),
421+
(None, False, "IGNORE", True),
422+
(None, False, "UPDATE", True),
423+
(None, False, "UPDATE", False),
424+
(None, False, "DEFAULT", True),
425+
(None, False, "DEFAULT", False),
426+
(None, True, "IGNORE", False),
427+
(None, True, "IGNORE", True),
428+
(None, True, "UPDATE", True),
429+
(None, True, "UPDATE", False),
430+
(None, True, "DEFAULT", True),
431+
(None, True, "DEFAULT", False),
432+
(10, False, "IGNORE", False),
433+
(10, False, "IGNORE", True),
434+
(10, False, "UPDATE", True),
435+
(10, False, "UPDATE", False),
436+
(10, False, "DEFAULT", True),
437+
(10, False, "DEFAULT", False),
438+
(10, True, "IGNORE", False),
439+
(10, True, "IGNORE", True),
440+
(10, True, "UPDATE", True),
441+
(10, True, "UPDATE", False),
442+
(10, True, "DEFAULT", True),
443+
(10, True, "DEFAULT", False),
444+
],
389445
)
390446
def test_transfer_specific_tables_transfers_only_specified_tables_from_sqlite_file(
391447
self,
@@ -397,6 +453,8 @@ def test_transfer_specific_tables_transfers_only_specified_tables_from_sqlite_fi
397453
caplog,
398454
chunk,
399455
with_rowid,
456+
mysql_insert_method,
457+
ignore_duplicate_keys,
400458
):
401459
sqlite_engine = create_engine(
402460
"sqlite:///{database}".format(database=sqlite_database),
@@ -425,6 +483,8 @@ def test_transfer_specific_tables_transfers_only_specified_tables_from_sqlite_fi
425483
mysql_database=mysql_credentials.database,
426484
chunk=chunk,
427485
with_rowid=with_rowid,
486+
mysql_insert_method=mysql_insert_method,
487+
ignore_duplicate_keys=ignore_duplicate_keys,
428488
)
429489
caplog.set_level(logging.DEBUG)
430490
proc.transfer()

0 commit comments

Comments
 (0)