✨ handle generated columns (#65)

Author: techouse

sqlite3_to_mysql/sqlite_utils.py

@@ -9,6 +9,7 @@
 import six
 from pytimeparse.timeparse import timeparse
 from unidecode import unidecode
+from packaging import version
 
 if version_info.major == 3 and 4 <= version_info.minor <= 6:
     from backports.datetime_fromisoformat import MonkeyPatch  # pylint: disable=E0401
@@ -65,3 +66,13 @@ def convert_date(value):
         raise ValueError(  # pylint: disable=W0707
             "DATE field contains Invalid isoformat string: {}".format(err)
         )
+
+
+def check_sqlite_table_xinfo_support(version_string):
+    """Check for SQLite table_xinfo support."""
+    sqlite_version = version.parse(version_string)
+    if sqlite_version.major > 3 or (
+        sqlite_version.major == 3 and sqlite_version.minor >= 26
+    ):
+        return True
+    return False

sqlite3_to_mysql/transporter.py

@@ -30,6 +30,7 @@
     convert_decimal,
     convert_timedelta,
     unicase_compare,
+    check_sqlite_table_xinfo_support,
 )
 from .mysql_utils import (
     MYSQL_BLOB_COLUMN_TYPES,
@@ -157,6 +158,11 @@ def __init__(self, **kwargs):
 
         self._sqlite_cur = self._sqlite.cursor()
 
+        self._sqlite_version = self._get_sqlite_version()
+        self._sqlite_table_xinfo_support = check_sqlite_table_xinfo_support(
+            self._sqlite_version
+        )
+
         try:
             self._mysql = mysql.connector.connect(
                 user=self._mysql_user,
@@ -225,6 +231,17 @@ def _get_mysql_version(self):
             )
             raise
 
+    def _get_sqlite_version(self):
+        try:
+            self._sqlite_cur.execute("SELECT sqlite_version()")
+            return self._sqlite_cur.fetchone()[0]
+        except (IndexError, sqlite3.Error) as err:
+            self._logger.error(
+                "SQLite failed checking for InnoDB version: %s",
+                err,
+            )
+            raise
+
     def _sqlite_table_has_rowid(self, table):
         try:
             self._sqlite_cur.execute("""SELECT rowid FROM "{}" LIMIT 1""".format(table))
@@ -323,7 +340,14 @@ def _create_table(self, table_name, transfer_rowid=False):
         if transfer_rowid:
             sql += " `rowid` BIGINT NOT NULL, "
 
-        self._sqlite_cur.execute('PRAGMA table_info("{}")'.format(table_name))
+        self._sqlite_cur.execute(
+            'PRAGMA {command}("{table_name}")'.format(
+                command="table_xinfo"
+                if self._sqlite_table_xinfo_support
+                else "table_info",
+                table_name=table_name,
+            )
+        )
 
         rows = self._sqlite_cur.fetchall()
         compound_primary_key = (
@@ -335,6 +359,12 @@ def _create_table(self, table_name, transfer_rowid=False):
             mysql_safe_name = safe_identifier_length(column["name"])
             column_type = self._translate_type_from_sqlite_to_mysql(column["type"])
 
+            # The "hidden" value is 0 for visible columns, 1 for "hidden" columns,
+            # 2 for computed virtual columns and 3 for computed stored columns.
+            # Read more on hidden columns here https://www.sqlite.org/pragma.html#pragma_table_xinfo
+            if "hidden" in column and column["hidden"] == 1:
+                continue
+
             sql += " `{name}` {type} {notnull} {default} {auto_increment}, ".format(
                 name=mysql_safe_name,
                 type=column_type,