✨ Transfer JSON columns as JSON (#22)

Author: techouse

README.md

@@ -38,39 +38,34 @@ Options:
   -u, --mysql-user TEXT           MySQL user  [required]
   -p, --prompt-mysql-password     Prompt for MySQL password
   --mysql-password TEXT           MySQL password
-  -t, --mysql-tables TEXT         Transfer only these specific tables (space
+  -t, --mysql-tables TUPLE        Transfer only these specific tables (space
                                   separated table names). Implies --without-
                                   foreign-keys which inhibits the transfer of
                                   foreign keys.
-
   -L, --limit-rows INTEGER        Transfer only a limited number of rows from
                                   each table.
-
   -C, --collation [BINARY|NOCASE|RTRIM]
                                   Create datatypes of TEXT affinity using a
                                   specified collation sequence.  [default:
                                   BINARY]
-
   -K, --prefix-indices            Prefix indices with their corresponding
                                   tables. This ensures that their names remain
                                   unique across the SQLite database.
-
   -X, --without-foreign-keys      Do not transfer foreign keys.
   -h, --mysql-host TEXT           MySQL host. Defaults to localhost.
   -P, --mysql-port INTEGER        MySQL port. Defaults to 3306.
   -S, --skip-ssl                  Disable MySQL connection encryption.
   -c, --chunk INTEGER             Chunk reading/writing SQL records
   -l, --log-file PATH             Log file
+  --json-as-text                  Transfer JSON columns as TEXT.
   -V, --vacuum                    Use the VACUUM command to rebuild the SQLite
                                   database file, repacking it into a minimal
                                   amount of disk space
-
   --use-buffered-cursors          Use MySQLCursorBuffered for reading the
                                   MySQL database. This can be useful in
                                   situations where multiple queries, with
                                   small result sets, need to be combined or
                                   computed with each other.
-
   -q, --quiet                     Quiet. Display only errors.
   --version                       Show the version and exit.
   --help                          Show this message and exit.

mysql_to_sqlite3/cli.py

@@ -90,6 +90,7 @@
     help="Chunk reading/writing SQL records",
 )
 @click.option("-l", "--log-file", type=click.Path(), help="Log file")
+@click.option("--json-as-text", is_flag=True, help="Transfer JSON columns as TEXT.")
 @click.option(
     "-V",
     "--vacuum",
@@ -124,6 +125,7 @@ def cli(
     skip_ssl,
     chunk,
     log_file,
+    json_as_text,
     vacuum,
     use_buffered_cursors,
     quiet,
@@ -145,6 +147,7 @@ def cli(
             mysql_port=mysql_port,
             mysql_ssl_disabled=skip_ssl,
             chunk=chunk,
+            json_as_text=json_as_text,
             vacuum=vacuum,
             buffered=use_buffered_cursors,
             log_file=log_file,

mysql_to_sqlite3/transporter.py

@@ -109,6 +109,11 @@ def __init__(self, **kwargs):
 
         self._sqlite_cur = self._sqlite.cursor()
 
+        self._json_as_text = kwargs.get("json_as_text") or False
+        self._sqlite_json1_extension_enabled = (
+            not self._json_as_text and self._check_sqlite_json1_extension_enabled()
+        )
+
         try:
             self._mysql = mysql.connector.connect(
                 user=self._mysql_user,
@@ -170,7 +175,9 @@ def _column_type_length(cls, column_type):
         return ""
 
     @classmethod
-    def _translate_type_from_mysql_to_sqlite(cls, column_type):
+    def _translate_type_from_mysql_to_sqlite(
+        cls, column_type, sqlite_json1_extension_enabled=False
+    ):
         """Handle MySQL 8."""
         try:
             column_type = column_type.decode()
@@ -223,6 +230,8 @@ def _translate_type_from_mysql_to_sqlite(cls, column_type):
             return "INTEGER"
         if data_type in "TIMESTAMP":
             return "DATETIME"
+        if data_type == "JSON" and sqlite_json1_extension_enabled:
+            return "JSON"
         return "TEXT"
 
     @classmethod
@@ -273,6 +282,13 @@ def _data_type_collation_sequence(
                 return "COLLATE {collation}".format(collation=collation)
         return ""
 
+    def _check_sqlite_json1_extension_enabled(self):
+        try:
+            self._sqlite_cur.execute("PRAGMA compile_options")
+            return "ENABLE_JSON1" in set(row[0] for row in self._sqlite_cur.fetchall())
+        except sqlite3.Error:
+            return False
+
     def _build_create_table_sql(self, table_name):
         sql = 'CREATE TABLE IF NOT EXISTS "{}" ('.format(table_name)
         primary = ""
@@ -281,7 +297,10 @@ def _build_create_table_sql(self, table_name):
         self._mysql_cur_dict.execute("SHOW COLUMNS FROM `{}`".format(table_name))
 
         for row in self._mysql_cur_dict.fetchall():
-            column_type = self._translate_type_from_mysql_to_sqlite(row["Type"])
+            column_type = self._translate_type_from_mysql_to_sqlite(
+                column_type=row["Type"],
+                sqlite_json1_extension_enabled=self._sqlite_json1_extension_enabled,
+            )
             sql += '\n\t"{name}" {type} {notnull} {default} {collation},'.format(
                 name=row["Field"],
                 type=column_type,

tests/unit/mysql_to_sqlite3_test.py

@@ -16,7 +16,7 @@ class TestMySQLtoSQLiteClassmethods:
     def test_translate_type_from_mysql_to_sqlite_invalid_column_type(self, mocker):
         with pytest.raises(ValueError) as excinfo:
             mocker.patch.object(MySQLtoSQLite, "_valid_column_type", return_value=False)
-            MySQLtoSQLite._translate_type_from_mysql_to_sqlite("text")
+            MySQLtoSQLite._translate_type_from_mysql_to_sqlite(column_type="text")
         assert "Invalid column_type!" in str(excinfo.value)
 
     def test_translate_type_from_mysql_to_sqlite_all_valid_columns(self):
@@ -99,7 +99,6 @@ def test_translate_type_from_mysql_to_sqlite_all_valid_columns(self):
                 )
             elif column_type in {
                 "ENUM",
-                "JSON",
                 "LONGTEXT",
                 "MEDIUMTEXT",
                 "SET",
@@ -109,6 +108,17 @@ def test_translate_type_from_mysql_to_sqlite_all_valid_columns(self):
                     MySQLtoSQLite._translate_type_from_mysql_to_sqlite(column_type)
                     == "TEXT"
                 )
+            elif column_type == "JSON":
+                assert (
+                    MySQLtoSQLite._translate_type_from_mysql_to_sqlite(column_type)
+                    == "TEXT"
+                )
+                assert (
+                    MySQLtoSQLite._translate_type_from_mysql_to_sqlite(
+                        column_type, sqlite_json1_extension_enabled=True
+                    )
+                    == "JSON"
+                )
             elif column_type.endswith(" UNSIGNED"):
                 if column_type.startswith("INT "):
                     assert (

tox.ini

@@ -57,7 +57,7 @@ skip_install = true
 deps =
     pylint
     -rrequirements_dev.txt
-disable = C0301,C0411,R,W0107,W0622
+disable = C0209,C0301,C0411,R,W0107,W0622
 commands =
     pylint --rcfile=tox.ini mysql_to_sqlite3