Skip to content

Commit d6c619c

Browse files
authored
✨ add --mysql-skip-create-tables and --mysql-skip-transfer-data options (#117)
1 parent b0569ce commit d6c619c

File tree

7 files changed

+195
-42
lines changed

7 files changed

+195
-42
lines changed

README.md

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ sqlite3mysql --help
3232
```
3333
Usage: sqlite3mysql [OPTIONS]
3434
35-
Transfer SQLite to MySQL using the provided CLI options.
35+
sqlite3mysql version 2.1.10 Copyright (c) 2018-2024 Klemen Tusar
3636
3737
Options:
3838
-f, --sqlite-file PATH SQLite3 database file [required]
@@ -52,7 +52,7 @@ Options:
5252
-h, --mysql-host TEXT MySQL host. Defaults to localhost.
5353
-P, --mysql-port INTEGER MySQL port. Defaults to 3306.
5454
-S, --skip-ssl Disable MySQL connection encryption.
55-
-i, --mysql-insert-method [UPDATE|IGNORE|DEFAULT]
55+
-i, --mysql-insert-method [DEFAULT|IGNORE|UPDATE]
5656
MySQL insert method. DEFAULT will throw
5757
errors when encountering duplicate records;
5858
UPDATE will update existing rows; IGNORE
@@ -64,7 +64,7 @@ Options:
6464
to INT(11).
6565
--mysql-string-type TEXT MySQL default string field type. Defaults to
6666
VARCHAR(255).
67-
--mysql-text-type [MEDIUMTEXT|TEXT|TINYTEXT|LONGTEXT]
67+
--mysql-text-type [LONGTEXT|MEDIUMTEXT|TEXT|TINYTEXT]
6868
MySQL default text field type. Defaults to
6969
TEXT.
7070
--mysql-charset TEXT MySQL database and table character set
@@ -75,6 +75,8 @@ Options:
7575
not support InnoDB FULLTEXT indexes!
7676
--with-rowid Transfer rowid columns.
7777
-c, --chunk INTEGER Chunk reading/writing SQL records
78+
-K, --mysql-skip-create-tables Skip creating tables in MySQL.
79+
-J, --mysql-skip-transfer-data Skip transferring data to MySQL.
7880
-l, --log-file PATH Log file
7981
-q, --quiet Quiet. Display only errors.
8082
--debug Debug mode. Will throw exceptions.

docs/README.rst

Lines changed: 19 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -23,37 +23,40 @@ Password Options
2323
- ``-p, --prompt-mysql-password``: Prompt for MySQL password
2424
- ``--mysql-password TEXT``: MySQL password
2525

26-
Table Options
27-
""""""""""""""
26+
Connection Options
27+
""""""""""""""""""
2828

29-
- ``-t, --sqlite-tables TUPLE``: Transfer only these specific tables (space separated table names). Implies ``--without-foreign-keys`` which inhibits the transfer of foreign keys.
30-
- ``-X, --without-foreign-keys``: Do not transfer foreign keys.
31-
- ``-W, --ignore-duplicate-keys``: Ignore duplicate keys. The default behavior is to create new ones with a numerical suffix, e.g. 'existing_key' -> 'existing_key_1'
32-
- ``-E, --mysql-truncate-tables``: Truncates existing tables before inserting data.
29+
- ``-h, --mysql-host TEXT``: MySQL host. Defaults to localhost.
30+
- ``-P, --mysql-port INTEGER``: MySQL port. Defaults to 3306.
31+
- ``-S, --skip-ssl``: Disable MySQL connection encryption.
3332

3433
Transfer Options
3534
""""""""""""""""
3635

36+
- ``-t, --sqlite-tables TUPLE``: Transfer only these specific tables (space separated table names). Implies ``--without-foreign-keys`` which inhibits the transfer of foreign keys.
37+
- ``-E, --mysql-truncate-tables``: Truncates existing tables before inserting data.
38+
- ``-K, --mysql-skip-create-tables``: Skip creating tables in MySQL.
3739
- ``-i, --mysql-insert-method [UPDATE|IGNORE|DEFAULT]``: MySQL insert method. DEFAULT will throw errors when encountering duplicate records; UPDATE will update existing rows; IGNORE will ignore insert errors. Defaults to IGNORE.
40+
- ``-J, --mysql-skip-transfer-data``: Skip transferring data to MySQL.
41+
- ``--mysql-integer-type TEXT``: MySQL default integer field type. Defaults to INT(11).
42+
- ``--mysql-string-type TEXT``: MySQL default string field type. Defaults to VARCHAR(255).
43+
- ``--mysql-text-type [LONGTEXT|MEDIUMTEXT|TEXT|TINYTEXT]``: MySQL default text field type. Defaults to TEXT.
44+
- ``--mysql-charset TEXT``: MySQL database and table character set. Defaults to utf8mb4.
45+
` ``--mysql-collation TEXT``: MySQL database and table collation
46+
- ``-T, --use-fulltext``: Use FULLTEXT indexes on TEXT columns. Will throw an error if your MySQL version does not support InnoDB FULLTEXT indexes!
47+
- ``-X, --without-foreign-keys``: Do not transfer foreign keys.
48+
- ``-W, --ignore-duplicate-keys``: Ignore duplicate keys. The default behavior is to create new ones with a numerical suffix, e.g. 'existing_key' -> 'existing_key_1'
3849
- ``--with-rowid``: Transfer rowid columns.
3950
- ``-c, --chunk INTEGER``: Chunk reading/writing SQL records
4051

41-
Connection Options
42-
""""""""""""""""""
43-
44-
- ``-h, --mysql-host TEXT``: MySQL host. Defaults to localhost.
45-
- ``-P, --mysql-port INTEGER``: MySQL port. Defaults to 3306.
46-
- ``-S, --skip-ssl``: Disable MySQL connection encryption.
47-
4852
Other Options
49-
""""""""""""""
53+
"""""""""""""
5054

51-
- ``-T, --use-fulltext``: Use FULLTEXT indexes on TEXT columns. Will throw an error if your MySQL version does not support InnoDB FULLTEXT indexes!
5255
- ``-l, --log-file PATH``: Log file
5356
- ``-q, --quiet``: Quiet. Display only errors.
5457
- ``--debug``: Debug mode. Will throw exceptions.
5558
- ``--version``: Show the version and exit.
56-
- ``--help``: Show this message and exit.
59+
- ``--help``: Show help message and exit.
5760

5861
Docker
5962
^^^^^^

src/sqlite3_to_mysql/cli.py

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,8 @@
120120
)
121121
@click.option("--with-rowid", is_flag=True, help="Transfer rowid columns.")
122122
@click.option("-c", "--chunk", type=int, default=None, help="Chunk reading/writing SQL records")
123+
@click.option("-K", "--mysql-skip-create-tables", is_flag=True, help="Skip creating tables in MySQL.")
124+
@click.option("-J", "--mysql-skip-transfer-data", is_flag=True, help="Skip transferring data to MySQL.")
123125
@click.option("-l", "--log-file", type=click.Path(), help="Log file")
124126
@click.option("-q", "--quiet", is_flag=True, help="Quiet. Display only errors.")
125127
@click.option("--debug", is_flag=True, help="Debug mode. Will throw exceptions.")
@@ -146,6 +148,8 @@ def cli(
146148
use_fulltext: bool,
147149
with_rowid: bool,
148150
chunk: int,
151+
mysql_skip_create_tables: bool,
152+
mysql_skip_transfer_data: bool,
149153
log_file: t.Union[str, "os.PathLike[t.Any]"],
150154
quiet: bool,
151155
debug: bool,
@@ -159,9 +163,17 @@ def cli(
159163
)
160164
if mysql_collation not in set(charset_collations):
161165
raise click.ClickException(
162-
f"""Error: Invalid value for '--collation' of charset '{mysql_charset}': '{mysql_collation}' is not one of {"'" + "', '".join(charset_collations) + "'"}."""
166+
f"Error: Invalid value for '--collation' of charset '{mysql_charset}': '{mysql_collation}' "
167+
f"""is not one of {"'" + "', '".join(charset_collations) + "'"}."""
163168
)
164169

170+
# check if both mysql_skip_create_table and mysql_skip_transfer_data are True
171+
if mysql_skip_create_tables and mysql_skip_transfer_data:
172+
raise click.ClickException(
173+
"Error: Both -K/--mysql-skip-create-tables and -J/--mysql-skip-transfer-data are set. "
174+
"There is nothing to do. Exiting..."
175+
)
176+
165177
SQLite3toMySQL(
166178
sqlite_file=sqlite_file,
167179
sqlite_tables=sqlite_tables or tuple(),
@@ -183,6 +195,8 @@ def cli(
183195
use_fulltext=use_fulltext,
184196
with_rowid=with_rowid,
185197
chunk=chunk,
198+
mysql_create_tables=not mysql_skip_create_tables,
199+
mysql_transfer_data=not mysql_skip_transfer_data,
186200
log_file=log_file,
187201
quiet=quiet,
188202
).transfer()

src/sqlite3_to_mysql/transporter.py

Lines changed: 33 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -69,51 +69,51 @@ def __init__(self, **kwargs: tx.Unpack[SQLite3toMySQLParams]):
6969

7070
self._mysql_password = str(kwargs.get("mysql_password")) or None
7171

72-
self._mysql_host = kwargs.get("mysql_host") or "localhost"
72+
self._mysql_host = str(kwargs.get("mysql_host", "localhost"))
7373

74-
self._mysql_port = kwargs.get("mysql_port") or 3306
74+
self._mysql_port = kwargs.get("mysql_port", 3306) or 3306
7575

7676
self._sqlite_tables = kwargs.get("sqlite_tables") or tuple()
7777

78-
self._without_foreign_keys = len(self._sqlite_tables) > 0 or kwargs.get("without_foreign_keys") or False
78+
self._without_foreign_keys = bool(self._sqlite_tables) or bool(kwargs.get("without_foreign_keys", False))
7979

80-
self._mysql_ssl_disabled = kwargs.get("mysql_ssl_disabled") or False
80+
self._mysql_ssl_disabled = bool(kwargs.get("mysql_ssl_disabled", False))
8181

82-
self._chunk_size = kwargs.get("chunk") or None
82+
self._chunk_size = bool(kwargs.get("chunk"))
8383

84-
self._quiet = kwargs.get("quiet") or False
84+
self._quiet = bool(kwargs.get("quiet", False))
8585

86-
self._logger = self._setup_logger(log_file=kwargs.get("log_file") or None, quiet=self._quiet)
86+
self._logger = self._setup_logger(log_file=kwargs.get("log_file", None), quiet=self._quiet)
8787

88-
self._mysql_database = kwargs.get("mysql_database") or "transfer"
88+
self._mysql_database = kwargs.get("mysql_database", "transfer") or "transfer"
8989

90-
self._mysql_insert_method = str(kwargs.get("mysql_integer_type") or "IGNORE").upper()
90+
self._mysql_insert_method = str(kwargs.get("mysql_integer_type", "IGNORE")).upper()
9191
if self._mysql_insert_method not in MYSQL_INSERT_METHOD:
9292
self._mysql_insert_method = "IGNORE"
9393

94-
self._mysql_truncate_tables = kwargs.get("mysql_truncate_tables") or False
94+
self._mysql_truncate_tables = bool(kwargs.get("mysql_truncate_tables", False))
9595

96-
self._mysql_integer_type = str(kwargs.get("mysql_integer_type") or "INT(11)").upper()
96+
self._mysql_integer_type = str(kwargs.get("mysql_integer_type", "INT(11)")).upper()
9797

98-
self._mysql_string_type = str(kwargs.get("mysql_string_type") or "VARCHAR(255)").upper()
98+
self._mysql_string_type = str(kwargs.get("mysql_string_type", "VARCHAR(255)")).upper()
9999

100-
self._mysql_text_type = str(kwargs.get("mysql_text_type") or "TEXT").upper()
100+
self._mysql_text_type = str(kwargs.get("mysql_text_type", "TEXT")).upper()
101101
if self._mysql_text_type not in MYSQL_TEXT_COLUMN_TYPES:
102102
self._mysql_text_type = "TEXT"
103103

104-
self._mysql_charset = kwargs.get("mysql_charset") or "utf8mb4"
104+
self._mysql_charset = kwargs.get("mysql_charset", "utf8mb4") or "utf8mb4"
105105

106106
self._mysql_collation = (
107107
kwargs.get("mysql_collation") or CharacterSet().get_default_collation(self._mysql_charset.lower())[0]
108108
)
109109
if not kwargs.get("mysql_collation") and self._mysql_collation == "utf8mb4_0900_ai_ci":
110110
self._mysql_collation = "utf8mb4_general_ci"
111111

112-
self._ignore_duplicate_keys = kwargs.get("ignore_duplicate_keys") or False
112+
self._ignore_duplicate_keys = kwargs.get("ignore_duplicate_keys", False) or False
113113

114-
self._use_fulltext = kwargs.get("use_fulltext") or False
114+
self._use_fulltext = kwargs.get("use_fulltext", False) or False
115115

116-
self._with_rowid = kwargs.get("with_rowid") or False
116+
self._with_rowid = kwargs.get("with_rowid", False) or False
117117

118118
sqlite3.register_adapter(Decimal, adapt_decimal)
119119
sqlite3.register_converter("DECIMAL", convert_decimal)
@@ -130,6 +130,12 @@ def __init__(self, **kwargs: tx.Unpack[SQLite3toMySQLParams]):
130130
self._sqlite_version = self._get_sqlite_version()
131131
self._sqlite_table_xinfo_support = check_sqlite_table_xinfo_support(self._sqlite_version)
132132

133+
self._mysql_create_tables = bool(kwargs.get("mysql_create_tables", True))
134+
self._mysql_transfer_data = bool(kwargs.get("mysql_transfer_data", True))
135+
136+
if not self._mysql_transfer_data and not self._mysql_create_tables:
137+
raise ValueError("Unable to continue without transferring data or creating tables!")
138+
133139
try:
134140
_mysql_connection = mysql.connector.connect(
135141
user=self._mysql_user,
@@ -677,15 +683,19 @@ def transfer(self) -> None:
677683
transfer_rowid: bool = self._with_rowid and self._sqlite_table_has_rowid(table["name"])
678684

679685
# create the table
680-
self._create_table(table["name"], transfer_rowid=transfer_rowid)
686+
if self._mysql_create_tables:
687+
self._create_table(table["name"], transfer_rowid=transfer_rowid)
681688

682689
# truncate the table on request
683690
if self._mysql_truncate_tables:
684691
self._truncate_table(table["name"])
685692

686693
# get the size of the data
687-
self._sqlite_cur.execute(f'SELECT COUNT(*) AS total_records FROM "{table["name"]}"')
688-
total_records = int(dict(self._sqlite_cur.fetchone())["total_records"])
694+
if self._mysql_transfer_data:
695+
self._sqlite_cur.execute(f'SELECT COUNT(*) AS total_records FROM "{table["name"]}"')
696+
total_records = int(dict(self._sqlite_cur.fetchone())["total_records"])
697+
else:
698+
total_records = 0
689699

690700
# only continue if there is anything to transfer
691701
if total_records > 0:
@@ -738,10 +748,11 @@ def transfer(self) -> None:
738748
raise
739749

740750
# add indices
741-
self._add_indices(table["name"])
751+
if self._mysql_create_tables:
752+
self._add_indices(table["name"])
742753

743754
# add foreign keys
744-
if not self._without_foreign_keys:
755+
if self._mysql_create_tables and not self._without_foreign_keys:
745756
self._add_foreign_keys(table["name"])
746757
except Exception: # pylint: disable=W0706
747758
raise

src/sqlite3_to_mysql/types.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,9 @@ class SQLite3toMySQLParams(tx.TypedDict):
2626
log_file: t.Optional[t.Union[str, "os.PathLike[t.Any]"]]
2727
mysql_database: t.Optional[str]
2828
mysql_integer_type: t.Optional[str]
29+
mysql_create_tables: t.Optional[bool]
2930
mysql_truncate_tables: t.Optional[bool]
31+
mysql_transfer_data: t.Optional[bool]
3032
mysql_charset: t.Optional[str]
3133
mysql_collation: t.Optional[str]
3234
ignore_duplicate_keys: t.Optional[bool]
@@ -54,7 +56,9 @@ class SQLite3toMySQLAttributes:
5456
_log_file: t.Union[str, "os.PathLike[t.Any]"]
5557
_mysql_database: str
5658
_mysql_insert_method: str
59+
_mysql_create_tables: bool
5760
_mysql_truncate_tables: bool
61+
_mysql_transfer_data: bool
5862
_mysql_integer_type: str
5963
_mysql_string_type: str
6064
_mysql_text_type: str

tests/func/sqlite3_to_mysql_test.py

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,34 @@ def test_bad_mysql_connection(
200200
)
201201
assert "Unable to connect to MySQL" in str(excinfo.value)
202202

203+
@pytest.mark.init
204+
@pytest.mark.parametrize("quiet", [False, True])
205+
def test_mysql_skip_create_tables_and_transfer_data(
206+
self,
207+
sqlite_database: str,
208+
mysql_credentials: MySQLCredentials,
209+
mocker: MockFixture,
210+
quiet: bool,
211+
) -> None:
212+
mocker.patch.object(
213+
SQLite3toMySQL,
214+
"transfer",
215+
return_value=None,
216+
)
217+
with pytest.raises(ValueError) as excinfo:
218+
SQLite3toMySQL( # type: ignore[call-arg]
219+
sqlite_file=sqlite_database,
220+
mysql_user=mysql_credentials.user,
221+
mysql_password=mysql_credentials.password,
222+
mysql_host=mysql_credentials.host,
223+
mysql_port=mysql_credentials.port,
224+
mysql_database=mysql_credentials.database,
225+
mysql_create_tables=False,
226+
mysql_transfer_data=False,
227+
quiet=quiet,
228+
)
229+
assert "Unable to continue without transferring data or creating tables!" in str(excinfo.value)
230+
203231
@pytest.mark.xfail
204232
@pytest.mark.init
205233
@pytest.mark.parametrize("quiet", [False, True])

0 commit comments

Comments
 (0)