✨ add --mysql-skip-create-tables and --mysql-skip-transfer-data options (#117)

Author: techouse

README.md

@@ -32,7 +32,7 @@ sqlite3mysql --help
 ```
 Usage: sqlite3mysql [OPTIONS]
 
-  Transfer SQLite to MySQL using the provided CLI options.
+  sqlite3mysql version 2.1.10 Copyright (c) 2018-2024 Klemen Tusar
 
 Options:
   -f, --sqlite-file PATH          SQLite3 database file  [required]
@@ -52,7 +52,7 @@ Options:
   -h, --mysql-host TEXT           MySQL host. Defaults to localhost.
   -P, --mysql-port INTEGER        MySQL port. Defaults to 3306.
   -S, --skip-ssl                  Disable MySQL connection encryption.
-  -i, --mysql-insert-method [UPDATE|IGNORE|DEFAULT]
+  -i, --mysql-insert-method [DEFAULT|IGNORE|UPDATE]
                                   MySQL insert method. DEFAULT will throw
                                   errors when encountering duplicate records;
                                   UPDATE will update existing rows; IGNORE
@@ -64,7 +64,7 @@ Options:
                                   to INT(11).
   --mysql-string-type TEXT        MySQL default string field type. Defaults to
                                   VARCHAR(255).
-  --mysql-text-type [MEDIUMTEXT|TEXT|TINYTEXT|LONGTEXT]
+  --mysql-text-type [LONGTEXT|MEDIUMTEXT|TEXT|TINYTEXT]
                                   MySQL default text field type. Defaults to
                                   TEXT.
   --mysql-charset TEXT            MySQL database and table character set
@@ -75,6 +75,8 @@ Options:
                                   not support InnoDB FULLTEXT indexes!
   --with-rowid                    Transfer rowid columns.
   -c, --chunk INTEGER             Chunk reading/writing SQL records
+  -K, --mysql-skip-create-tables  Skip creating tables in MySQL.
+  -J, --mysql-skip-transfer-data  Skip transferring data to MySQL.
   -l, --log-file PATH             Log file
   -q, --quiet                     Quiet. Display only errors.
   --debug                         Debug mode. Will throw exceptions.

docs/README.rst

@@ -23,37 +23,40 @@ Password Options
 - ``-p, --prompt-mysql-password``: Prompt for MySQL password
 - ``--mysql-password TEXT``: MySQL password
 
-Table Options
-""""""""""""""
+Connection Options
+""""""""""""""""""
 
-- ``-t, --sqlite-tables TUPLE``: Transfer only these specific tables (space separated table names). Implies ``--without-foreign-keys`` which inhibits the transfer of foreign keys.
-- ``-X, --without-foreign-keys``: Do not transfer foreign keys.
-- ``-W, --ignore-duplicate-keys``: Ignore duplicate keys. The default behavior is to create new ones with a numerical suffix, e.g. 'existing_key' -> 'existing_key_1'
-- ``-E, --mysql-truncate-tables``: Truncates existing tables before inserting data.
+- ``-h, --mysql-host TEXT``: MySQL host. Defaults to localhost.
+- ``-P, --mysql-port INTEGER``: MySQL port. Defaults to 3306.
+- ``-S, --skip-ssl``: Disable MySQL connection encryption.
 
 Transfer Options
 """"""""""""""""
 
+- ``-t, --sqlite-tables TUPLE``: Transfer only these specific tables (space separated table names). Implies ``--without-foreign-keys`` which inhibits the transfer of foreign keys.
+- ``-E, --mysql-truncate-tables``: Truncates existing tables before inserting data.
+- ``-K, --mysql-skip-create-tables``: Skip creating tables in MySQL.
 - ``-i, --mysql-insert-method [UPDATE|IGNORE|DEFAULT]``: MySQL insert method. DEFAULT will throw errors when encountering duplicate records; UPDATE will update existing rows; IGNORE will ignore insert errors. Defaults to IGNORE.
+- ``-J, --mysql-skip-transfer-data``: Skip transferring data to MySQL.
+- ``--mysql-integer-type TEXT``: MySQL default integer field type. Defaults to INT(11).
+- ``--mysql-string-type TEXT``: MySQL default string field type. Defaults to VARCHAR(255).
+- ``--mysql-text-type [LONGTEXT|MEDIUMTEXT|TEXT|TINYTEXT]``: MySQL default text field type. Defaults to TEXT.
+- ``--mysql-charset TEXT``: MySQL database and table character set. Defaults to utf8mb4.
+` ``--mysql-collation TEXT``: MySQL database and table collation
+- ``-T, --use-fulltext``: Use FULLTEXT indexes on TEXT columns. Will throw an error if your MySQL version does not support InnoDB FULLTEXT indexes!
+- ``-X, --without-foreign-keys``: Do not transfer foreign keys.
+- ``-W, --ignore-duplicate-keys``: Ignore duplicate keys. The default behavior is to create new ones with a numerical suffix, e.g. 'existing_key' -> 'existing_key_1'
 - ``--with-rowid``: Transfer rowid columns.
 - ``-c, --chunk INTEGER``: Chunk reading/writing SQL records
 
-Connection Options
-""""""""""""""""""
-
-- ``-h, --mysql-host TEXT``: MySQL host. Defaults to localhost.
-- ``-P, --mysql-port INTEGER``: MySQL port. Defaults to 3306.
-- ``-S, --skip-ssl``: Disable MySQL connection encryption.
-
 Other Options
-""""""""""""""
+"""""""""""""
 
-- ``-T, --use-fulltext``: Use FULLTEXT indexes on TEXT columns. Will throw an error if your MySQL version does not support InnoDB FULLTEXT indexes!
 - ``-l, --log-file PATH``: Log file
 - ``-q, --quiet``: Quiet. Display only errors.
 - ``--debug``: Debug mode. Will throw exceptions.
 - ``--version``: Show the version and exit.
-- ``--help``: Show this message and exit.
+- ``--help``: Show help message and exit.
 
 Docker
 ^^^^^^

src/sqlite3_to_mysql/cli.py

@@ -120,6 +120,8 @@
 )
 @click.option("--with-rowid", is_flag=True, help="Transfer rowid columns.")
 @click.option("-c", "--chunk", type=int, default=None, help="Chunk reading/writing SQL records")
+@click.option("-K", "--mysql-skip-create-tables", is_flag=True, help="Skip creating tables in MySQL.")
+@click.option("-J", "--mysql-skip-transfer-data", is_flag=True, help="Skip transferring data to MySQL.")
 @click.option("-l", "--log-file", type=click.Path(), help="Log file")
 @click.option("-q", "--quiet", is_flag=True, help="Quiet. Display only errors.")
 @click.option("--debug", is_flag=True, help="Debug mode. Will throw exceptions.")
@@ -146,6 +148,8 @@ def cli(
     use_fulltext: bool,
     with_rowid: bool,
     chunk: int,
+    mysql_skip_create_tables: bool,
+    mysql_skip_transfer_data: bool,
     log_file: t.Union[str, "os.PathLike[t.Any]"],
     quiet: bool,
     debug: bool,
@@ -159,9 +163,17 @@ def cli(
             )
             if mysql_collation not in set(charset_collations):
                 raise click.ClickException(
-                    f"""Error: Invalid value for '--collation' of charset '{mysql_charset}': '{mysql_collation}' is not one of {"'" + "', '".join(charset_collations) + "'"}."""
+                    f"Error: Invalid value for '--collation' of charset '{mysql_charset}': '{mysql_collation}' "
+                    f"""is not one of {"'" + "', '".join(charset_collations) + "'"}."""
                 )
 
+        # check if both mysql_skip_create_table and mysql_skip_transfer_data are True
+        if mysql_skip_create_tables and mysql_skip_transfer_data:
+            raise click.ClickException(
+                "Error: Both -K/--mysql-skip-create-tables and -J/--mysql-skip-transfer-data are set. "
+                "There is nothing to do. Exiting..."
+            )
+
         SQLite3toMySQL(
             sqlite_file=sqlite_file,
             sqlite_tables=sqlite_tables or tuple(),
@@ -183,6 +195,8 @@ def cli(
             use_fulltext=use_fulltext,
             with_rowid=with_rowid,
             chunk=chunk,
+            mysql_create_tables=not mysql_skip_create_tables,
+            mysql_transfer_data=not mysql_skip_transfer_data,
             log_file=log_file,
             quiet=quiet,
         ).transfer()

src/sqlite3_to_mysql/transporter.py

@@ -69,51 +69,51 @@ def __init__(self, **kwargs: tx.Unpack[SQLite3toMySQLParams]):
 
         self._mysql_password = str(kwargs.get("mysql_password")) or None
 
-        self._mysql_host = kwargs.get("mysql_host") or "localhost"
+        self._mysql_host = str(kwargs.get("mysql_host", "localhost"))
 
-        self._mysql_port = kwargs.get("mysql_port") or 3306
+        self._mysql_port = kwargs.get("mysql_port", 3306) or 3306
 
         self._sqlite_tables = kwargs.get("sqlite_tables") or tuple()
 
-        self._without_foreign_keys = len(self._sqlite_tables) > 0 or kwargs.get("without_foreign_keys") or False
+        self._without_foreign_keys = bool(self._sqlite_tables) or bool(kwargs.get("without_foreign_keys", False))
 
-        self._mysql_ssl_disabled = kwargs.get("mysql_ssl_disabled") or False
+        self._mysql_ssl_disabled = bool(kwargs.get("mysql_ssl_disabled", False))
 
-        self._chunk_size = kwargs.get("chunk") or None
+        self._chunk_size = bool(kwargs.get("chunk"))
 
-        self._quiet = kwargs.get("quiet") or False
+        self._quiet = bool(kwargs.get("quiet", False))
 
-        self._logger = self._setup_logger(log_file=kwargs.get("log_file") or None, quiet=self._quiet)
+        self._logger = self._setup_logger(log_file=kwargs.get("log_file", None), quiet=self._quiet)
 
-        self._mysql_database = kwargs.get("mysql_database") or "transfer"
+        self._mysql_database = kwargs.get("mysql_database", "transfer") or "transfer"
 
-        self._mysql_insert_method = str(kwargs.get("mysql_integer_type") or "IGNORE").upper()
+        self._mysql_insert_method = str(kwargs.get("mysql_integer_type", "IGNORE")).upper()
         if self._mysql_insert_method not in MYSQL_INSERT_METHOD:
             self._mysql_insert_method = "IGNORE"
 
-        self._mysql_truncate_tables = kwargs.get("mysql_truncate_tables") or False
+        self._mysql_truncate_tables = bool(kwargs.get("mysql_truncate_tables", False))
 
-        self._mysql_integer_type = str(kwargs.get("mysql_integer_type") or "INT(11)").upper()
+        self._mysql_integer_type = str(kwargs.get("mysql_integer_type", "INT(11)")).upper()
 
-        self._mysql_string_type = str(kwargs.get("mysql_string_type") or "VARCHAR(255)").upper()
+        self._mysql_string_type = str(kwargs.get("mysql_string_type", "VARCHAR(255)")).upper()
 
-        self._mysql_text_type = str(kwargs.get("mysql_text_type") or "TEXT").upper()
+        self._mysql_text_type = str(kwargs.get("mysql_text_type", "TEXT")).upper()
         if self._mysql_text_type not in MYSQL_TEXT_COLUMN_TYPES:
             self._mysql_text_type = "TEXT"
 
-        self._mysql_charset = kwargs.get("mysql_charset") or "utf8mb4"
+        self._mysql_charset = kwargs.get("mysql_charset", "utf8mb4") or "utf8mb4"
 
         self._mysql_collation = (
             kwargs.get("mysql_collation") or CharacterSet().get_default_collation(self._mysql_charset.lower())[0]
         )
         if not kwargs.get("mysql_collation") and self._mysql_collation == "utf8mb4_0900_ai_ci":
             self._mysql_collation = "utf8mb4_general_ci"
 
-        self._ignore_duplicate_keys = kwargs.get("ignore_duplicate_keys") or False
+        self._ignore_duplicate_keys = kwargs.get("ignore_duplicate_keys", False) or False
 
-        self._use_fulltext = kwargs.get("use_fulltext") or False
+        self._use_fulltext = kwargs.get("use_fulltext", False) or False
 
-        self._with_rowid = kwargs.get("with_rowid") or False
+        self._with_rowid = kwargs.get("with_rowid", False) or False
 
         sqlite3.register_adapter(Decimal, adapt_decimal)
         sqlite3.register_converter("DECIMAL", convert_decimal)
@@ -130,6 +130,12 @@ def __init__(self, **kwargs: tx.Unpack[SQLite3toMySQLParams]):
         self._sqlite_version = self._get_sqlite_version()
         self._sqlite_table_xinfo_support = check_sqlite_table_xinfo_support(self._sqlite_version)
 
+        self._mysql_create_tables = bool(kwargs.get("mysql_create_tables", True))
+        self._mysql_transfer_data = bool(kwargs.get("mysql_transfer_data", True))
+
+        if not self._mysql_transfer_data and not self._mysql_create_tables:
+            raise ValueError("Unable to continue without transferring data or creating tables!")
+
         try:
             _mysql_connection = mysql.connector.connect(
                 user=self._mysql_user,
@@ -677,15 +683,19 @@ def transfer(self) -> None:
                 transfer_rowid: bool = self._with_rowid and self._sqlite_table_has_rowid(table["name"])
 
                 # create the table
-                self._create_table(table["name"], transfer_rowid=transfer_rowid)
+                if self._mysql_create_tables:
+                    self._create_table(table["name"], transfer_rowid=transfer_rowid)
 
                 # truncate the table on request
                 if self._mysql_truncate_tables:
                     self._truncate_table(table["name"])
 
                 # get the size of the data
-                self._sqlite_cur.execute(f'SELECT COUNT(*) AS total_records FROM "{table["name"]}"')
-                total_records = int(dict(self._sqlite_cur.fetchone())["total_records"])
+                if self._mysql_transfer_data:
+                    self._sqlite_cur.execute(f'SELECT COUNT(*) AS total_records FROM "{table["name"]}"')
+                    total_records = int(dict(self._sqlite_cur.fetchone())["total_records"])
+                else:
+                    total_records = 0
 
                 # only continue if there is anything to transfer
                 if total_records > 0:
@@ -738,10 +748,11 @@ def transfer(self) -> None:
                         raise
 
                 # add indices
-                self._add_indices(table["name"])
+                if self._mysql_create_tables:
+                    self._add_indices(table["name"])
 
                 # add foreign keys
-                if not self._without_foreign_keys:
+                if self._mysql_create_tables and not self._without_foreign_keys:
                     self._add_foreign_keys(table["name"])
         except Exception:  # pylint: disable=W0706
             raise

src/sqlite3_to_mysql/types.py

@@ -26,7 +26,9 @@ class SQLite3toMySQLParams(tx.TypedDict):
     log_file: t.Optional[t.Union[str, "os.PathLike[t.Any]"]]
     mysql_database: t.Optional[str]
     mysql_integer_type: t.Optional[str]
+    mysql_create_tables: t.Optional[bool]
     mysql_truncate_tables: t.Optional[bool]
+    mysql_transfer_data: t.Optional[bool]
     mysql_charset: t.Optional[str]
     mysql_collation: t.Optional[str]
     ignore_duplicate_keys: t.Optional[bool]
@@ -54,7 +56,9 @@ class SQLite3toMySQLAttributes:
     _log_file: t.Union[str, "os.PathLike[t.Any]"]
     _mysql_database: str
     _mysql_insert_method: str
+    _mysql_create_tables: bool
     _mysql_truncate_tables: bool
+    _mysql_transfer_data: bool
     _mysql_integer_type: str
     _mysql_string_type: str
     _mysql_text_type: str

tests/func/sqlite3_to_mysql_test.py

@@ -200,6 +200,34 @@ def test_bad_mysql_connection(
             )
         assert "Unable to connect to MySQL" in str(excinfo.value)
 
+    @pytest.mark.init
+    @pytest.mark.parametrize("quiet", [False, True])
+    def test_mysql_skip_create_tables_and_transfer_data(
+        self,
+        sqlite_database: str,
+        mysql_credentials: MySQLCredentials,
+        mocker: MockFixture,
+        quiet: bool,
+    ) -> None:
+        mocker.patch.object(
+            SQLite3toMySQL,
+            "transfer",
+            return_value=None,
+        )
+        with pytest.raises(ValueError) as excinfo:
+            SQLite3toMySQL(  # type: ignore[call-arg]
+                sqlite_file=sqlite_database,
+                mysql_user=mysql_credentials.user,
+                mysql_password=mysql_credentials.password,
+                mysql_host=mysql_credentials.host,
+                mysql_port=mysql_credentials.port,
+                mysql_database=mysql_credentials.database,
+                mysql_create_tables=False,
+                mysql_transfer_data=False,
+                quiet=quiet,
+            )
+        assert "Unable to continue without transferring data or creating tables!" in str(excinfo.value)
+
     @pytest.mark.xfail
     @pytest.mark.init
     @pytest.mark.parametrize("quiet", [False, True])