Skip to content

Commit 885b258

Browse files
techousetechouse
authored
Merge pull request #49 from techouse/feat/issue-48-charset-introducers-in-default
✨ Add support for MySQL character set introducers in DEFAULT clause
2 parents 75f4330 + 23fd261 commit 885b258

File tree

4 files changed

+112
-20
lines changed

4 files changed

+112
-20
lines changed

‎mysql_to_sqlite3/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
"""Utility to transfer data from MySQL to SQLite 3."""
2-
__version__ = "2.0.0"
2+
__version__ = "2.0.1"
33

44
from .transporter import MySQLtoSQLite

‎mysql_to_sqlite3/mysql_utils.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
"""Miscellaneous MySQL utilities."""
2+
3+
import typing as t
4+
5+
from mysql.connector.charsets import MYSQL_CHARACTER_SETS
6+
7+
8+
CHARSET_INTRODUCERS: t.Tuple[str, ...] = tuple(
9+
f"_{charset[0]}" for charset in MYSQL_CHARACTER_SETS if charset is not None
10+
)

‎mysql_to_sqlite3/transporter.py

Lines changed: 72 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
from mysql.connector.types import ToPythonOutputTypes
1818
from tqdm import tqdm, trange
1919

20+
from mysql_to_sqlite3.mysql_utils import CHARSET_INTRODUCERS
2021
from mysql_to_sqlite3.sqlite_utils import (
2122
CollatingSequences,
2223
adapt_decimal,
@@ -249,8 +250,13 @@ def _translate_type_from_mysql_to_sqlite(
249250

250251
@classmethod
251252
def _translate_default_from_mysql_to_sqlite(
252-
cls, column_default: t.Optional[t.Any] = None, column_type: t.Optional[str] = None
253+
cls,
254+
column_default: ToPythonOutputTypes = None,
255+
column_type: t.Optional[str] = None,
256+
column_extra: ToPythonOutputTypes = None,
253257
) -> str:
258+
is_binary: bool
259+
is_hex: bool
254260
if isinstance(column_default, bytes):
255261
if column_type in {
256262
"BIT",
@@ -261,6 +267,34 @@ def _translate_default_from_mysql_to_sqlite(
261267
"TINYBLOB",
262268
"VARBINARY",
263269
}:
270+
if column_extra in {"DEFAULT_GENERATED", "default_generated"}:
271+
for charset_introducer in CHARSET_INTRODUCERS:
272+
if column_default.startswith(charset_introducer.encode()):
273+
is_binary = False
274+
is_hex = False
275+
for b_prefix in ("B", "b"):
276+
if column_default.startswith(rf"{charset_introducer} {b_prefix}\'".encode()):
277+
is_binary = True
278+
break
279+
for x_prefix in ("X", "x"):
280+
if column_default.startswith(rf"{charset_introducer} {x_prefix}\'".encode()):
281+
is_hex = True
282+
break
283+
column_default = (
284+
column_default.replace(charset_introducer.encode(), b"")
285+
.replace(rb"x\'", b"")
286+
.replace(rb"X\'", b"")
287+
.replace(rb"b\'", b"")
288+
.replace(rb"B\'", b"")
289+
.replace(rb"\'", b"")
290+
.replace(rb"'", b"")
291+
.strip()
292+
)
293+
if is_binary:
294+
return f"DEFAULT '{chr(int(column_default, 2))}'"
295+
if is_hex:
296+
return f"DEFAULT x'{column_default.decode()}'"
297+
break
264298
return f"DEFAULT x'{column_default.hex()}'"
265299
try:
266300
column_default = column_default.decode()
@@ -275,13 +309,42 @@ def _translate_default_from_mysql_to_sqlite(
275309
return "DEFAULT(FALSE)"
276310
return f"DEFAULT '{int(column_default)}'"
277311
if isinstance(column_default, str):
278-
if column_default.upper() in {
279-
"CURRENT_TIME",
280-
"CURRENT_DATE",
281-
"CURRENT_TIMESTAMP",
282-
}:
283-
return f"DEFAULT {column_default.upper()}"
284-
return f"DEFAULT '{str(column_default)}'"
312+
if column_extra in {"DEFAULT_GENERATED", "default_generated"}:
313+
if column_default.upper() in {
314+
"CURRENT_TIME",
315+
"CURRENT_DATE",
316+
"CURRENT_TIMESTAMP",
317+
}:
318+
return f"DEFAULT {column_default.upper()}"
319+
for charset_introducer in CHARSET_INTRODUCERS:
320+
if column_default.startswith(charset_introducer):
321+
is_binary = False
322+
is_hex = False
323+
for b_prefix in ("B", "b"):
324+
if column_default.startswith(rf"{charset_introducer} {b_prefix}\'"):
325+
is_binary = True
326+
break
327+
for x_prefix in ("X", "x"):
328+
if column_default.startswith(rf"{charset_introducer} {x_prefix}\'"):
329+
is_hex = True
330+
break
331+
column_default = (
332+
column_default.replace(charset_introducer, "")
333+
.replace(r"x\'", "")
334+
.replace(r"X\'", "")
335+
.replace(r"b\'", "")
336+
.replace(r"B\'", "")
337+
.replace(r"\'", "")
338+
.replace(r"'", "")
339+
.strip()
340+
)
341+
if is_binary:
342+
return f"DEFAULT '{chr(int(column_default, 2))}'"
343+
if is_hex:
344+
return f"DEFAULT x'{column_default}'"
345+
return f"DEFAULT '{column_default}'"
346+
return "DEFAULT '{}'".format(column_default.replace(r"\'", r"''"))
347+
return "DEFAULT '{}'".format(str(column_default).replace(r"\'", r"''"))
285348

286349
@classmethod
287350
def _data_type_collation_sequence(
@@ -324,7 +387,7 @@ def _build_create_table_sql(self, table_name: str) -> str:
324387
name=row["Field"].decode() if isinstance(row["Field"], bytes) else row["Field"],
325388
type=column_type,
326389
notnull="NULL" if row["Null"] == "YES" else "NOT NULL",
327-
default=self._translate_default_from_mysql_to_sqlite(row["Default"], column_type),
390+
default=self._translate_default_from_mysql_to_sqlite(row["Default"], column_type, row["Extra"]),
328391
collation=self._data_type_collation_sequence(self._collation, column_type),
329392
)
330393

‎tests/unit/mysql_to_sqlite3_test.py

Lines changed: 29 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -98,38 +98,57 @@ def test_translate_type_from_mysql_to_sqlite_all_valid_columns(self) -> None:
9898
assert MySQLtoSQLite._translate_type_from_mysql_to_sqlite(column_type) == column_type
9999

100100
@pytest.mark.parametrize(
101-
"column_default, sqlite_default_translation",
101+
"column_default, column_extra, sqlite_default_translation",
102102
[
103-
pytest.param(None, "", id="None"),
104-
pytest.param("", "DEFAULT ''", id='""'),
105-
pytest.param("lorem", "DEFAULT 'lorem'", id='"lorem"'),
103+
pytest.param(None, None, "", id="None"),
104+
pytest.param("", None, "DEFAULT ''", id='""'),
105+
pytest.param("lorem", None, "DEFAULT 'lorem'", id='"lorem"'),
106106
pytest.param(
107107
"lorem ipsum dolor",
108+
None,
108109
"DEFAULT 'lorem ipsum dolor'",
109110
id='"lorem ipsum dolor"',
110111
),
111-
pytest.param("CURRENT_TIME", "DEFAULT CURRENT_TIME", id='"CURRENT_TIME"'),
112-
pytest.param("current_time", "DEFAULT CURRENT_TIME", id='"current_time"'),
113-
pytest.param("CURRENT_DATE", "DEFAULT CURRENT_DATE", id='"CURRENT_DATE"'),
114-
pytest.param("current_date", "DEFAULT CURRENT_DATE", id='"current_date"'),
112+
pytest.param("CURRENT_TIME", "DEFAULT_GENERATED", "DEFAULT CURRENT_TIME", id='"CURRENT_TIME"'),
113+
pytest.param("current_time", "DEFAULT_GENERATED", "DEFAULT CURRENT_TIME", id='"current_time"'),
114+
pytest.param("CURRENT_DATE", "DEFAULT_GENERATED", "DEFAULT CURRENT_DATE", id='"CURRENT_DATE"'),
115+
pytest.param("current_date", "DEFAULT_GENERATED", "DEFAULT CURRENT_DATE", id='"current_date"'),
115116
pytest.param(
116117
"CURRENT_TIMESTAMP",
118+
"DEFAULT_GENERATED",
117119
"DEFAULT CURRENT_TIMESTAMP",
118120
id='"CURRENT_TIMESTAMP"',
119121
),
120122
pytest.param(
121123
"current_timestamp",
124+
"DEFAULT_GENERATED",
122125
"DEFAULT CURRENT_TIMESTAMP",
123126
id='"current_timestamp"',
124127
),
128+
pytest.param(r"""_utf8mb4\'[]\'""", "DEFAULT_GENERATED", "DEFAULT '[]'", id=r"""_utf8mb4\'[]\'"""),
129+
pytest.param(r"""_latin1\'abc\'""", "DEFAULT_GENERATED", "DEFAULT 'abc'", id=r"""_latin1\'abc\'"""),
130+
pytest.param(r"""_binary\'abc\'""", "DEFAULT_GENERATED", "DEFAULT 'abc'", id=r"""_binary\'abc\'"""),
131+
pytest.param(
132+
r"""_latin1 X\'4D7953514C\'""",
133+
"DEFAULT_GENERATED",
134+
"DEFAULT x'4D7953514C'",
135+
id=r"""_latin1 X\'4D7953514C\'""",
136+
),
137+
pytest.param(
138+
r"""_latin1 b\'1000001\'""", "DEFAULT_GENERATED", "DEFAULT 'A'", id=r"""_latin1 b\'1000001\'"""
139+
),
125140
],
126141
)
127142
def test_translate_default_from_mysql_to_sqlite(
128143
self,
129-
column_default: str,
144+
column_default: t.Optional[str],
145+
column_extra: t.Optional[str],
130146
sqlite_default_translation: str,
131147
) -> None:
132-
assert MySQLtoSQLite._translate_default_from_mysql_to_sqlite(column_default) == sqlite_default_translation
148+
assert (
149+
MySQLtoSQLite._translate_default_from_mysql_to_sqlite(column_default, column_extra=column_extra)
150+
== sqlite_default_translation
151+
)
133152

134153
@pytest.mark.parametrize(
135154
"column_default, sqlite_default_translation, sqlite_version",

0 commit comments

Comments
 (0)