diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 6f4c1602a5e64..98e816ac9f067 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -60,10 +60,31 @@ pytest.mark.filterwarnings( "ignore:Passing a BlockManager to DataFrame:DeprecationWarning" ), - pytest.mark.single_cpu, + pytest.mark.db, ] +def create_uuid(): + return lambda uuid_val: f"{uuid_val}_" + f"{uuid.uuid4().hex}"[0:10] + + +def repeat(constant): + while True: + yield constant + + +def setup(connection_variables, table_uuid=None, uuid_views=None): + conn = [] + for each in connection_variables: + if type(each) == type("str"): + conn.append(each) + else: + conn.append(each[0][0]) + return list( + map(lambda *args: args, conn, repeat(table_uuid), repeat(uuid_views)) + ) + + @pytest.fixture def sql_strings(): return { @@ -93,7 +114,7 @@ def sql_strings(): } -def iris_table_metadata(): +def iris_table_metadata(iris_name: str): import sqlalchemy from sqlalchemy import ( Column, @@ -107,7 +128,7 @@ def iris_table_metadata(): dtype = Double if Version(sqlalchemy.__version__) >= Version("2.0.0") else Float metadata = MetaData() iris = Table( - "iris", + iris_name, metadata, Column("SepalLength", dtype), Column("SepalWidth", dtype), @@ -118,23 +139,25 @@ def iris_table_metadata(): return iris -def create_and_load_iris_sqlite3(conn, iris_file: Path): - stmt = """CREATE TABLE iris ( +def create_and_load_iris_sqlite3(conn, iris_file: Path, iris_uuid): + stmt = f"""CREATE TABLE {iris_uuid} ( "SepalLength" REAL, "SepalWidth" REAL, "PetalLength" REAL, "PetalWidth" REAL, "Name" TEXT )""" + sqlalchemy = pytest.importorskip("sqlalchemy") + if type(conn) == sqlalchemy.engine.base.Engine: + conn = conn.raw_connection() cur = conn.cursor() cur.execute(stmt) with iris_file.open(newline=None, encoding="utf-8") as csvfile: reader = csv.reader(csvfile) next(reader) - stmt = "INSERT INTO iris VALUES(?, ?, ?, ?, ?)" + stmt = f"INSERT INTO {iris_uuid} VALUES(?, ?, ?, ?, ?)" # ADBC requires explicit types - no implicit str -> float conversion - records = [] records = [ ( float(row[0]), @@ -152,8 +175,8 @@ def create_and_load_iris_sqlite3(conn, iris_file: Path): conn.commit() -def create_and_load_iris_postgresql(conn, iris_file: Path): - stmt = """CREATE TABLE iris ( +def create_and_load_iris_postgresql(conn, iris_file: Path, iris_name): + stmt = f"""CREATE TABLE {iris_name} ( "SepalLength" DOUBLE PRECISION, "SepalWidth" DOUBLE PRECISION, "PetalLength" DOUBLE PRECISION, @@ -165,7 +188,7 @@ def create_and_load_iris_postgresql(conn, iris_file: Path): with iris_file.open(newline=None, encoding="utf-8") as csvfile: reader = csv.reader(csvfile) next(reader) - stmt = "INSERT INTO iris VALUES($1, $2, $3, $4, $5)" + stmt = f"INSERT INTO {iris_name} VALUES($1, $2, $3, $4, $5)" # ADBC requires explicit types - no implicit str -> float conversion records = [ ( @@ -183,29 +206,37 @@ def create_and_load_iris_postgresql(conn, iris_file: Path): conn.commit() -def create_and_load_iris(conn, iris_file: Path): +def create_and_load_iris(conn, iris_file: Path, iris_table_uuid: str): from sqlalchemy import insert + import sqlalchemy - iris = iris_table_metadata() + iris = iris_table_metadata(iris_table_uuid) with iris_file.open(newline=None, encoding="utf-8") as csvfile: reader = csv.reader(csvfile) header = next(reader) params = [dict(zip(header, row)) for row in reader] stmt = insert(iris).values(params) - with conn.begin() as con: - iris.drop(con, checkfirst=True) - iris.create(bind=con) - con.execute(stmt) + if type(conn) == sqlalchemy.engine.base.Engine: + with conn.begin() as con: + iris.drop(con, checkfirst=True) + iris.create(bind=con) + con.execute(stmt) + elif type(conn) == sqlalchemy.engine.base.Connection: + iris.drop(conn, checkfirst=True) + iris.create(bind=conn) + conn.execute(stmt) -def create_and_load_iris_view(conn): - stmt = "CREATE VIEW iris_view AS SELECT * FROM iris" +def create_and_load_iris_view(conn, iris_table_uuid, iris_view_uuid): + stmt = f"CREATE VIEW {iris_view_uuid} AS SELECT * FROM {iris_table_uuid}" if isinstance(conn, sqlite3.Connection): cur = conn.cursor() cur.execute(stmt) else: - adbc = import_optional_dependency("adbc_driver_manager.dbapi", errors="ignore") + adbc = import_optional_dependency( + "adbc_driver_manager.dbapi", errors="ignore" + ) if adbc and isinstance(conn, adbc.Connection): with conn.cursor() as cur: cur.execute(stmt) @@ -218,7 +249,7 @@ def create_and_load_iris_view(conn): con.execute(stmt) -def types_table_metadata(dialect: str): +def types_table_metadata(dialect: str, table_name): from sqlalchemy import ( TEXT, Boolean, @@ -234,7 +265,7 @@ def types_table_metadata(dialect: str): bool_type = Integer if dialect == "sqlite" else Boolean metadata = MetaData() types = Table( - "types", + table_name, metadata, Column("TextCol", TEXT), # error: Cannot infer type argument 1 of "Column" @@ -252,8 +283,8 @@ def types_table_metadata(dialect: str): return types -def create_and_load_types_sqlite3(conn, types_data: list[dict]): - stmt = """CREATE TABLE types ( +def create_and_load_types_sqlite3(conn, types_data: list[dict], table_uuid): + stmt = f"""CREATE TABLE {table_uuid} ( "TextCol" TEXT, "DateCol" TEXT, "IntDateCol" INTEGER, @@ -265,8 +296,8 @@ def create_and_load_types_sqlite3(conn, types_data: list[dict]): "BoolColWithNull" INTEGER )""" - ins_stmt = """ - INSERT INTO types + ins_stmt = f""" + INSERT INTO {table_uuid} VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?) """ @@ -282,9 +313,9 @@ def create_and_load_types_sqlite3(conn, types_data: list[dict]): conn.commit() -def create_and_load_types_postgresql(conn, types_data: list[dict]): +def create_and_load_types_postgresql(conn, types_data: list[dict], table_uuid): with conn.cursor() as cur: - stmt = """CREATE TABLE types ( + stmt = f"""CREATE TABLE {table_uuid} ( "TextCol" TEXT, "DateCol" TIMESTAMP, "IntDateCol" INTEGER, @@ -297,8 +328,8 @@ def create_and_load_types_postgresql(conn, types_data: list[dict]): )""" cur.execute(stmt) - stmt = """ - INSERT INTO types + stmt = f""" + INSERT INTO {table_uuid} VALUES($1, $2::timestamp, $3, $4, $5, $6, $7, $8, $9) """ @@ -307,11 +338,11 @@ def create_and_load_types_postgresql(conn, types_data: list[dict]): conn.commit() -def create_and_load_types(conn, types_data: list[dict], dialect: str): +def create_and_load_types(conn, types_data: list[dict], dialect: str, table_name): from sqlalchemy import insert from sqlalchemy.engine import Engine - types = types_table_metadata(dialect) + types = types_table_metadata(dialect, table_name) stmt = insert(types).values(types_data) if isinstance(conn, Engine): @@ -327,7 +358,7 @@ def create_and_load_types(conn, types_data: list[dict], dialect: str): conn.execute(stmt) -def create_and_load_postgres_datetz(conn): +def create_and_load_postgres_datetz(conn, table_name): from sqlalchemy import ( Column, DateTime, @@ -338,7 +369,9 @@ def create_and_load_postgres_datetz(conn): from sqlalchemy.engine import Engine metadata = MetaData() - datetz = Table("datetz", metadata, Column("DateColWithTz", DateTime(timezone=True))) + datetz = Table( + table_name, metadata, Column("DateColWithTz", DateTime(timezone=True)) + ) datetz_data = [ { "DateColWithTz": "2000-01-01 00:00:00-08:00", @@ -514,7 +547,9 @@ def get_all_views(conn): c = conn.execute("SELECT name FROM sqlite_master WHERE type='view'") return [view[0] for view in c.fetchall()] else: - adbc = import_optional_dependency("adbc_driver_manager.dbapi", errors="ignore") + adbc = import_optional_dependency( + "adbc_driver_manager.dbapi", errors="ignore" + ) if adbc and isinstance(conn, adbc.Connection): results = [] info = conn.adbc_get_objects().read_all().to_pylist() @@ -539,7 +574,9 @@ def get_all_tables(conn): c = conn.execute("SELECT name FROM sqlite_master WHERE type='table'") return [table[0] for table in c.fetchall()] else: - adbc = import_optional_dependency("adbc_driver_manager.dbapi", errors="ignore") + adbc = import_optional_dependency( + "adbc_driver_manager.dbapi", errors="ignore" + ) if adbc and isinstance(conn, adbc.Connection): results = [] @@ -560,44 +597,79 @@ def get_all_tables(conn): def drop_table( table_name: str, - conn: sqlite3.Connection | sqlalchemy.engine.Engine | sqlalchemy.engine.Connection, + conn: ( + sqlite3.Connection | sqlalchemy.engine.Engine | sqlalchemy.engine.Connection + ), ): if isinstance(conn, sqlite3.Connection): - conn.execute(f"DROP TABLE IF EXISTS {sql._get_valid_sqlite_name(table_name)}") + conn.execute( + f"DROP TABLE IF EXISTS {sql._get_valid_sqlite_name(table_name)}" + ) conn.commit() else: - adbc = import_optional_dependency("adbc_driver_manager.dbapi", errors="ignore") + adbc = import_optional_dependency( + "adbc_driver_manager.dbapi", errors="ignore" + ) if adbc and isinstance(conn, adbc.Connection): with conn.cursor() as cur: cur.execute(f'DROP TABLE IF EXISTS "{table_name}"') + conn.commit() else: - with conn.begin() as con: - with sql.SQLDatabase(con) as db: - db.drop_table(table_name) + import sqlalchemy + + # Better matching for dialect string literal tables + quoted_table_name = ( + conn.engine.dialect.identifier_preparer.quote_identifier(table_name) + ) + stmt = sqlalchemy.text(f"DROP TABLE IF EXISTS {quoted_table_name}") + + if isinstance(conn, sqlalchemy.Engine): + conn = conn.connect() + conn.commit() + with conn.begin(): + conn.execute(stmt) def drop_view( view_name: str, - conn: sqlite3.Connection | sqlalchemy.engine.Engine | sqlalchemy.engine.Connection, + conn: ( + sqlite3.Connection | sqlalchemy.engine.Engine | sqlalchemy.engine.Connection + ), ): - import sqlalchemy + sqlalchemy = pytest.importorskip("sqlalchemy") + from sqlalchemy import Engine if isinstance(conn, sqlite3.Connection): conn.execute(f"DROP VIEW IF EXISTS {sql._get_valid_sqlite_name(view_name)}") conn.commit() else: - adbc = import_optional_dependency("adbc_driver_manager.dbapi", errors="ignore") + adbc = import_optional_dependency( + "adbc_driver_manager.dbapi", errors="ignore" + ) if adbc and isinstance(conn, adbc.Connection): with conn.cursor() as cur: cur.execute(f'DROP VIEW IF EXISTS "{view_name}"') + conn.commit() else: quoted_view = conn.engine.dialect.identifier_preparer.quote_identifier( view_name ) stmt = sqlalchemy.text(f"DROP VIEW IF EXISTS {quoted_view}") - with conn.begin() as con: - con.execute(stmt) # type: ignore[union-attr] + if isinstance(conn, Engine): + conn = conn.connect() + if conn.in_transaction(): + conn.commit() + else: + with conn.begin(): + conn.execute(stmt) # type: ignore[union-attr] + conn.commit() + else: + if conn.in_transaction(): + conn.commit() + with conn.begin(): + conn.execute(stmt) # type: ignore[union-attr] + conn.commit() @pytest.fixture @@ -610,23 +682,28 @@ def mysql_pymysql_engine(): poolclass=sqlalchemy.pool.NullPool, ) yield engine - for view in get_all_views(engine): - drop_view(view, engine) - for tbl in get_all_tables(engine): - drop_table(tbl, engine) engine.dispose() @pytest.fixture -def mysql_pymysql_engine_iris(mysql_pymysql_engine, iris_path): - create_and_load_iris(mysql_pymysql_engine, iris_path) - create_and_load_iris_view(mysql_pymysql_engine) - return mysql_pymysql_engine +def mysql_pymysql_engine_iris(request, mysql_pymysql_engine, iris_path): + calling_test_name = request.node.name[0 : request.node.name.index("[")] + uuid_root = f"{calling_test_name}_" + f"{uuid.uuid4().hex}"[0:10] + iris_table_uuid = "tbl_" + uuid_root + iris_view_uuid = "view_" + uuid_root + + conn = mysql_pymysql_engine + + create_and_load_iris(conn, iris_path, iris_table_uuid) + create_and_load_iris_view(conn, iris_table_uuid, iris_view_uuid) + + yield conn, iris_table_uuid, iris_view_uuid + + conn.dispose() @pytest.fixture def mysql_pymysql_engine_types(mysql_pymysql_engine, types_data): - create_and_load_types(mysql_pymysql_engine, types_data, "mysql") return mysql_pymysql_engine @@ -634,18 +711,22 @@ def mysql_pymysql_engine_types(mysql_pymysql_engine, types_data): def mysql_pymysql_conn(mysql_pymysql_engine): with mysql_pymysql_engine.connect() as conn: yield conn + conn.close() @pytest.fixture def mysql_pymysql_conn_iris(mysql_pymysql_engine_iris): - with mysql_pymysql_engine_iris.connect() as conn: - yield conn + engine, iris_table_uuid, iris_view_uuid = mysql_pymysql_engine_iris + with engine.connect() as conn: + yield conn, iris_table_uuid, iris_view_uuid + engine.dispose() @pytest.fixture def mysql_pymysql_conn_types(mysql_pymysql_engine_types): with mysql_pymysql_engine_types.connect() as conn: yield conn + conn.close() @pytest.fixture @@ -656,24 +737,30 @@ def postgresql_psycopg2_engine(): "postgresql+psycopg2://postgres:postgres@localhost:5432/pandas", poolclass=sqlalchemy.pool.NullPool, ) + yield engine - for view in get_all_views(engine): - drop_view(view, engine) - for tbl in get_all_tables(engine): - drop_table(tbl, engine) + engine.dispose() @pytest.fixture -def postgresql_psycopg2_engine_iris(postgresql_psycopg2_engine, iris_path): - create_and_load_iris(postgresql_psycopg2_engine, iris_path) - create_and_load_iris_view(postgresql_psycopg2_engine) - return postgresql_psycopg2_engine +def postgresql_psycopg2_engine_iris(request, postgresql_psycopg2_engine, iris_path): + conn = postgresql_psycopg2_engine + + calling_test_name = request.node.name[0 : request.node.name.index("[")] + uuid_root = f"{calling_test_name}_" + f"{uuid.uuid4().hex}"[0:10] + iris_table_uuid = "tbl_" + uuid_root + iris_view_uuid = "view_" + uuid_root + + create_and_load_iris(conn, iris_path, iris_table_uuid) + create_and_load_iris_view(conn, iris_table_uuid, iris_view_uuid) + + yield conn, iris_table_uuid, iris_view_uuid + conn.dispose() @pytest.fixture def postgresql_psycopg2_engine_types(postgresql_psycopg2_engine, types_data): - create_and_load_types(postgresql_psycopg2_engine, types_data, "postgres") return postgresql_psycopg2_engine @@ -681,6 +768,7 @@ def postgresql_psycopg2_engine_types(postgresql_psycopg2_engine, types_data): def postgresql_psycopg2_conn(postgresql_psycopg2_engine): with postgresql_psycopg2_engine.connect() as conn: yield conn + conn.close() @pytest.fixture @@ -692,30 +780,24 @@ def postgresql_adbc_conn(): uri = "postgresql://postgres:postgres@localhost:5432/pandas" with dbapi.connect(uri) as conn: yield conn - for view in get_all_views(conn): - drop_view(view, conn) - for tbl in get_all_tables(conn): - drop_table(tbl, conn) - conn.commit() + conn.close() @pytest.fixture -def postgresql_adbc_iris(postgresql_adbc_conn, iris_path): +def postgresql_adbc_iris(request, postgresql_adbc_conn, iris_path): import adbc_driver_manager as mgr + calling_test_name = request.node.name[0 : request.node.name.index("[")] + uuid_root = f"{calling_test_name}_" + f"{uuid.uuid4().hex}"[0:10] + iris_table_uuid = "tbl_" + uuid_root + iris_view_uuid = "view_" + uuid_root + conn = postgresql_adbc_conn - try: - conn.adbc_get_table_schema("iris") - except mgr.ProgrammingError: - conn.rollback() - create_and_load_iris_postgresql(conn, iris_path) - try: - conn.adbc_get_table_schema("iris_view") - except mgr.ProgrammingError: # note arrow-adbc issue 1022 - conn.rollback() - create_and_load_iris_view(conn) - return conn + create_and_load_iris_postgresql(conn, iris_path, iris_table_uuid) + create_and_load_iris_view(conn, iris_table_uuid, iris_view_uuid) + + yield conn, iris_table_uuid, iris_view_uuid @pytest.fixture @@ -724,27 +806,21 @@ def postgresql_adbc_types(postgresql_adbc_conn, types_data): conn = postgresql_adbc_conn - try: - conn.adbc_get_table_schema("types") - except mgr.ProgrammingError: - conn.rollback() - new_data = [tuple(entry.values()) for entry in types_data] - - create_and_load_types_postgresql(conn, new_data) - return conn @pytest.fixture def postgresql_psycopg2_conn_iris(postgresql_psycopg2_engine_iris): - with postgresql_psycopg2_engine_iris.connect() as conn: - yield conn + engine, iris_table_uuid, iris_view_uuid = postgresql_psycopg2_engine_iris + with engine.connect() as conn: + yield conn, iris_table_uuid, iris_view_uuid @pytest.fixture def postgresql_psycopg2_conn_types(postgresql_psycopg2_engine_types): - with postgresql_psycopg2_engine_types.connect() as conn: - yield conn + conn = postgresql_psycopg2_engine_types.connect() + yield conn + conn.close() @pytest.fixture @@ -759,61 +835,77 @@ def sqlite_engine(sqlite_str): sqlalchemy = pytest.importorskip("sqlalchemy") engine = sqlalchemy.create_engine(sqlite_str, poolclass=sqlalchemy.pool.NullPool) yield engine - for view in get_all_views(engine): - drop_view(view, engine) - for tbl in get_all_tables(engine): - drop_table(tbl, engine) engine.dispose() @pytest.fixture def sqlite_conn(sqlite_engine): - with sqlite_engine.connect() as conn: + conn = sqlite_engine.connect() + try: yield conn + finally: + conn.close() @pytest.fixture -def sqlite_str_iris(sqlite_str, iris_path): +def sqlite_str_iris(request, iris_path, sqlite_str): sqlalchemy = pytest.importorskip("sqlalchemy") engine = sqlalchemy.create_engine(sqlite_str) - create_and_load_iris(engine, iris_path) - create_and_load_iris_view(engine) + + calling_test_name = request.node.name[0 : request.node.name.index("[")] + uuid_root = f"{calling_test_name}_" + f"{uuid.uuid4().hex}"[0:10] + iris_table_uuid = "tbl_" + uuid_root + iris_view_uuid = "view_" + uuid_root + + create_and_load_iris_sqlite3(engine, iris_path, iris_table_uuid) + create_and_load_iris_view(engine, iris_table_uuid, iris_view_uuid) engine.dispose() - return sqlite_str + yield sqlite_str, iris_table_uuid, iris_view_uuid @pytest.fixture -def sqlite_engine_iris(sqlite_engine, iris_path): - create_and_load_iris(sqlite_engine, iris_path) - create_and_load_iris_view(sqlite_engine) - return sqlite_engine +def sqlite_engine_iris(request, iris_path, sqlite_engine): + calling_test_name = request.node.name[0 : request.node.name.index("[")] + uuid_root = f"{calling_test_name}_" + f"{uuid.uuid4().hex}"[0:10] + iris_table_uuid = "tbl_" + uuid_root + iris_view_uuid = "view_" + uuid_root + engine = sqlite_engine + + create_and_load_iris_sqlite3(engine, iris_path, iris_table_uuid) + create_and_load_iris_view(engine, iris_table_uuid, iris_view_uuid) + yield engine, iris_table_uuid, iris_view_uuid @pytest.fixture def sqlite_conn_iris(sqlite_engine_iris): - with sqlite_engine_iris.connect() as conn: - yield conn + engine, iris_table_uuid, iris_view_uuid = sqlite_engine_iris + with engine.connect() as conn: + try: + yield conn, iris_table_uuid, iris_view_uuid + finally: + conn.close() @pytest.fixture def sqlite_str_types(sqlite_str, types_data): sqlalchemy = pytest.importorskip("sqlalchemy") engine = sqlalchemy.create_engine(sqlite_str) - create_and_load_types(engine, types_data, "sqlite") engine.dispose() return sqlite_str @pytest.fixture -def sqlite_engine_types(sqlite_engine, types_data): - create_and_load_types(sqlite_engine, types_data, "sqlite") +def sqlite_engine_types(sqlite_engine): return sqlite_engine @pytest.fixture def sqlite_conn_types(sqlite_engine_types): with sqlite_engine_types.connect() as conn: - yield conn + try: + yield conn + finally: + conn.close() @pytest.fixture @@ -822,33 +914,29 @@ def sqlite_adbc_conn(): pytest.importorskip("adbc_driver_sqlite") from adbc_driver_sqlite import dbapi - with tm.ensure_clean() as name: - uri = f"file:{name}" - with dbapi.connect(uri) as conn: - yield conn - for view in get_all_views(conn): - drop_view(view, conn) - for tbl in get_all_tables(conn): - drop_table(tbl, conn) - conn.commit() + try: + with tm.ensure_clean() as name: + uri = f"file:{name}" + with dbapi.connect(uri) as conn: + yield conn + finally: + conn.close() @pytest.fixture -def sqlite_adbc_iris(sqlite_adbc_conn, iris_path): +def sqlite_adbc_iris(request, sqlite_adbc_conn, iris_path): import adbc_driver_manager as mgr + calling_test_name = request.node.name[0 : request.node.name.index("[")] + uuid_root = f"{calling_test_name}_" + f"{uuid.uuid4().hex}"[0:10] + iris_table_uuid = "tbl_" + uuid_root + iris_view_uuid = "view_" + uuid_root + conn = sqlite_adbc_conn - try: - conn.adbc_get_table_schema("iris") - except mgr.ProgrammingError: - conn.rollback() - create_and_load_iris_sqlite3(conn, iris_path) - try: - conn.adbc_get_table_schema("iris_view") - except mgr.ProgrammingError: - conn.rollback() - create_and_load_iris_view(conn) - return conn + create_and_load_iris_sqlite3(conn, iris_path, iris_table_uuid) + create_and_load_iris_view(conn, iris_table_uuid, iris_view_uuid) + + yield conn, iris_table_uuid, iris_view_uuid @pytest.fixture @@ -856,41 +944,35 @@ def sqlite_adbc_types(sqlite_adbc_conn, types_data): import adbc_driver_manager as mgr conn = sqlite_adbc_conn - try: - conn.adbc_get_table_schema("types") - except mgr.ProgrammingError: - conn.rollback() - new_data = [] - for entry in types_data: - entry["BoolCol"] = int(entry["BoolCol"]) - if entry["BoolColWithNull"] is not None: - entry["BoolColWithNull"] = int(entry["BoolColWithNull"]) - new_data.append(tuple(entry.values())) - - create_and_load_types_sqlite3(conn, new_data) - conn.commit() return conn @pytest.fixture def sqlite_buildin(): - with contextlib.closing(sqlite3.connect(":memory:")) as closing_conn: - with closing_conn as conn: - yield conn + try: + conn = sqlite3.connect(":memory:") + yield conn + finally: + conn.close() @pytest.fixture -def sqlite_buildin_iris(sqlite_buildin, iris_path): - create_and_load_iris_sqlite3(sqlite_buildin, iris_path) - create_and_load_iris_view(sqlite_buildin) - return sqlite_buildin +def sqlite_buildin_iris(request, sqlite_buildin, iris_path): + calling_test_name = request.node.name[0 : request.node.name.index("[")] + uuid_root = f"{calling_test_name}_" + f"{uuid.uuid4().hex}"[0:10] + iris_table_uuid = "tbl_" + uuid_root + iris_view_uuid = "view_" + uuid_root + conn = sqlite_buildin + + create_and_load_iris_sqlite3(conn, iris_path, iris_table_uuid) + create_and_load_iris_view(conn, iris_table_uuid, iris_view_uuid) + yield conn, iris_table_uuid, iris_view_uuid @pytest.fixture def sqlite_buildin_types(sqlite_buildin, types_data): types_data = [tuple(entry.values()) for entry in types_data] - create_and_load_types_sqlite3(sqlite_buildin, types_data) return sqlite_buildin @@ -942,7 +1024,9 @@ def sqlite_buildin_types(sqlite_buildin, types_data): "sqlite_str_types", ] -sqlalchemy_connectable = mysql_connectable + postgresql_connectable + sqlite_connectable +sqlalchemy_connectable = ( + mysql_connectable + postgresql_connectable + sqlite_connectable +) sqlalchemy_connectable_iris = ( mysql_connectable_iris + postgresql_connectable_iris + sqlite_connectable_iris @@ -958,7 +1042,7 @@ def sqlite_buildin_types(sqlite_buildin, types_data): ] adbc_connectable_iris = [ - pytest.param("postgresql_adbc_iris", marks=pytest.mark.db), + pytest.param("sqlite_adbc_iris", marks=pytest.mark.db), "sqlite_adbc_iris", ] @@ -979,16 +1063,194 @@ def sqlite_buildin_types(sqlite_buildin, types_data): ) -@pytest.mark.parametrize("conn", all_connectable) -def test_dataframe_to_sql(conn, test_frame1, request): - # GH 51086 if conn is sqlite_engine - conn = request.getfixturevalue(conn) - test_frame1.to_sql(name="test", con=conn, if_exists="append", index=False) +@pytest.fixture +def iris_connect_and_per_test_id(request, iris_path): + sqlalchemy = pytest.importorskip("sqlalchemy") + conn_name = request.param[0] + + conn, table_uuid, view_uuid = request.getfixturevalue(conn_name) + yield { + "conn": conn, + "iris_table_uuid": table_uuid, + "iris_view_uuid": view_uuid, + "conn_name": conn_name, + } + if isinstance(conn, str): + conn = sqlalchemy.create_engine(conn) + drop_view(view_uuid, conn) + drop_table(table_uuid, conn) + conn.dispose() + else: + drop_view(view_uuid, conn) + drop_table(table_uuid, conn) + +connectables_to_create_uuid_function_map = { + "mysql_pymysql_engine_types": lambda eng, types_data, uuid: \ + create_and_load_types(eng, types_data, "mysql", uuid), -@pytest.mark.parametrize("conn", all_connectable) -def test_dataframe_to_sql_empty(conn, test_frame1, request): - if conn == "postgresql_adbc_conn" and not using_string_dtype(): + "mysql_pymysql_conn_types": lambda eng, types_data, uuid: \ + create_and_load_types(eng, types_data, "mysql", uuid), + + "postgresql_psycopg2_engine_types": lambda eng, types_data, uuid:\ + create_and_load_types( + eng, types_data, "postgres", uuid), + + "postgresql_adbc_types": lambda eng, new_data, uuid: \ + create_and_load_types_postgresql(eng, new_data, uuid), + + "postgresql_psycopg2_conn_types": lambda eng, types_data, uuid: \ + create_and_load_types(eng, types_data, "postgres", uuid), + + "sqlite_str_types": lambda eng, types_data, uuid: \ + create_and_load_types(eng, types_data, "sqlite", uuid), + + "sqlite_engine_types": lambda eng, types_data, uuid: \ + create_and_load_types(eng, types_data, "sqlite", uuid), + + "sqlite_conn_types": lambda eng, types_data, uuid: \ + create_and_load_types(eng, types_data, "sqlite", uuid), + + "sqlite_adbc_types": lambda eng, new_data, uuid: \ + create_and_load_types_sqlite3(eng, new_data, uuid), + + "sqlite_buildin_types": lambda eng, types_data, uuid: \ + create_and_load_types_sqlite3( + eng, + [tuple(entry.values()) for entry in types_data], + uuid + ), +} + + +def instantiate_types(conn, conn_name, table_uuid, types_data): + if conn_name in connectables_to_create_uuid_function_map.keys(): + # Load types function + types_connect_and_load = connectables_to_create_uuid_function_map[conn_name] + + # Prepare data to insert into types + if conn_name == "sqlite_adbc_types": + data = [] + for entry in types_data: + entry["BoolCol"] = int(entry["BoolCol"]) + if entry["BoolColWithNull"] is not None: + entry["BoolColWithNull"] = int(entry["BoolColWithNull"]) + data.append(tuple(entry.values())) + + elif conn_name == "postgresql_adbc_types": + data = [tuple(entry.values()) for entry in types_data] + + else: + data = types_data + + # Create engine if using sqlstring + if conn_name == "sqlite_str_types": + sqlalchemy = pytest.importorskip("sqlalchemy") + engine = sqlalchemy.create_engine(conn) + types_connect_and_load(engine, data, table_uuid) + engine.dispose() + else: + # Execute function that creates types_table + types_connect_and_load(conn, data, table_uuid) + + +def generate_uuid_strings(prefix, raw_uuid_value): + if raw_uuid_value is None: + return None + if isinstance(raw_uuid_value, str): + return prefix + "_" + create_uuid()(raw_uuid_value) + if isinstance(raw_uuid_value, list): + return [prefix + "_" + create_uuid()(p) for p in raw_uuid_value] + + +def drop_table_uuid_views(conn, table_uuid, view_uuid): + if view_uuid is not None: + if isinstance(view_uuid, list): + for view in view_uuid: + drop_view(view, conn) + else: + drop_view(view_uuid, conn) + + if table_uuid is not None: + if isinstance(table_uuid, list): + for table in table_uuid: + drop_table(table, conn) + else: + drop_table(table_uuid, conn) + + +@pytest.fixture +def connect_and_uuid_types(request, types_data): + + sqlalchemy = pytest.importorskip("sqlalchemy") + conn_name = request.param[0] + table_uuid = generate_uuid_strings("table", request.param[1]) + view_uuid = generate_uuid_strings("view", request.param[2]) + types_data = request.getfixturevalue("types_data") + conn = request.getfixturevalue(conn_name) + + instantiate_types(conn, conn_name, table_uuid, types_data) + + yield { + "conn": conn, + "table_uuid": table_uuid, + "view_uuid": view_uuid, + "conn_name": conn_name, + } + if isinstance(conn, str): + engine = sqlalchemy.create_engine(conn) + drop_table_uuid_views(engine, table_uuid, view_uuid) + engine.dispose() + else: + drop_table_uuid_views(conn, table_uuid, view_uuid) + + +@pytest.fixture +def connect_and_uuid(request, types_data): + sqlalchemy = pytest.importorskip("sqlalchemy") + conn_name = request.param[0] + + table_uuid = generate_uuid_strings("table", request.param[1]) + view_uuid = generate_uuid_strings("view", request.param[2]) + conn = request.getfixturevalue(conn_name) + + yield { + "conn": conn, + "table_uuid": table_uuid, + "view_uuid": view_uuid, + "conn_name": conn_name, + } + if isinstance(conn, str): + engine = sqlalchemy.create_engine(conn) + drop_table_uuid_views(engine, table_uuid, view_uuid) + engine.dispose() + else: + drop_table_uuid_views(conn, table_uuid, view_uuid) + + + + +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_dataframe_to_sql"), + indirect=True, +) +def test_dataframe_to_sql(connect_and_uuid, test_frame1, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + test_frame1.to_sql(name=table_uuid, con=conn, if_exists="append", index=False) + + +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_dataframe_to_sql_empty"), + indirect=True, +) +def test_dataframe_to_sql_empty(connect_and_uuid, test_frame1, request): + conn = connect_and_uuid["conn"] + conn_name = connect_and_uuid["conn_name"] + table_uuid = connect_and_uuid["table_uuid"] + if conn_name == "postgresql_adbc_conn" and not using_string_dtype(): request.node.add_marker( pytest.mark.xfail( reason="postgres ADBC driver < 1.2 cannot insert index with null type", @@ -996,14 +1258,21 @@ def test_dataframe_to_sql_empty(conn, test_frame1, request): ) # GH 51086 if conn is sqlite_engine - conn = request.getfixturevalue(conn) empty_df = test_frame1.iloc[:0] - empty_df.to_sql(name="test", con=conn, if_exists="append", index=False) + empty_df.to_sql(name=table_uuid, con=conn, if_exists="append", index=False) -@pytest.mark.parametrize("conn", all_connectable) -def test_dataframe_to_sql_arrow_dtypes(conn, request): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_dataframe_to_sql_arrow_dtypes"), + indirect=True, +) +def test_dataframe_to_sql_arrow_dtypes(connect_and_uuid, request): # GH 52046 + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + pytest.importorskip("pyarrow") df = DataFrame( { @@ -1017,8 +1286,8 @@ def test_dataframe_to_sql_arrow_dtypes(conn, request): } ) - if "adbc" in conn: - if conn == "sqlite_adbc_conn": + if "adbc" in conn_name: + if conn_name == "sqlite_adbc_conn": df = df.drop(columns=["timedelta"]) if pa_version_under14p1: exp_warning = DeprecationWarning @@ -1030,14 +1299,22 @@ def test_dataframe_to_sql_arrow_dtypes(conn, request): exp_warning = UserWarning msg = "the 'timedelta'" - conn = request.getfixturevalue(conn) with tm.assert_produces_warning(exp_warning, match=msg, check_stacklevel=False): - df.to_sql(name="test_arrow", con=conn, if_exists="replace", index=False) + df.to_sql(name=table_uuid, con=conn, if_exists="replace", index=False) -@pytest.mark.parametrize("conn", all_connectable) -def test_dataframe_to_sql_arrow_dtypes_missing(conn, request, nulls_fixture): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_dataframe_to_sql_arrow_dtypes_missing"), + indirect=True, +) +def test_dataframe_to_sql_arrow_dtypes_missing( + connect_and_uuid, request, nulls_fixture +): # GH 52046 + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + pytest.importorskip("pyarrow") if isinstance(nulls_fixture, Decimal): pytest.skip( @@ -1052,93 +1329,130 @@ def test_dataframe_to_sql_arrow_dtypes_missing(conn, request, nulls_fixture): ), } ) - conn = request.getfixturevalue(conn) - df.to_sql(name="test_arrow", con=conn, if_exists="replace", index=False) + df.to_sql(name=table_uuid, con=conn, if_exists="replace", index=False) -@pytest.mark.parametrize("conn", all_connectable) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_to_sql"), + indirect=True, +) @pytest.mark.parametrize("method", [None, "multi"]) -def test_to_sql(conn, method, test_frame1, request): - if method == "multi" and "adbc" in conn: +def test_to_sql(connect_and_uuid, method, test_frame1, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + + if method == "multi" and "adbc" in conn_name: request.node.add_marker( pytest.mark.xfail( reason="'method' not implemented for ADBC drivers", strict=True ) ) - conn = request.getfixturevalue(conn) with pandasSQL_builder(conn, need_transaction=True) as pandasSQL: - pandasSQL.to_sql(test_frame1, "test_frame", method=method) - assert pandasSQL.has_table("test_frame") - assert count_rows(conn, "test_frame") == len(test_frame1) + pandasSQL.to_sql(test_frame1, table_uuid, method=method) + assert pandasSQL.has_table(table_uuid) + assert count_rows(conn, table_uuid) == len(test_frame1) -@pytest.mark.parametrize("conn", all_connectable) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_to_sql_exist"), + indirect=True, +) @pytest.mark.parametrize( "mode, num_row_coef", [("replace", 1), ("append", 2), ("delete_rows", 1)] ) -def test_to_sql_exist(conn, mode, num_row_coef, test_frame1, request): - conn = request.getfixturevalue(conn) +def test_to_sql_exist(connect_and_uuid, mode, num_row_coef, test_frame1, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + with pandasSQL_builder(conn, need_transaction=True) as pandasSQL: - pandasSQL.to_sql(test_frame1, "test_frame", if_exists="fail") - pandasSQL.to_sql(test_frame1, "test_frame", if_exists=mode) - assert pandasSQL.has_table("test_frame") - assert count_rows(conn, "test_frame") == num_row_coef * len(test_frame1) + pandasSQL.to_sql(test_frame1, table_uuid, if_exists="fail") + pandasSQL.to_sql(test_frame1, table_uuid, if_exists=mode) + assert pandasSQL.has_table(table_uuid) + assert count_rows(conn, table_uuid) == num_row_coef * len(test_frame1) -@pytest.mark.parametrize("conn", all_connectable) -def test_to_sql_exist_fail(conn, test_frame1, request): - conn = request.getfixturevalue(conn) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_to_sql_exist_fail"), + indirect=True, +) +def test_to_sql_exist_fail(connect_and_uuid, test_frame1, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] with pandasSQL_builder(conn, need_transaction=True) as pandasSQL: - pandasSQL.to_sql(test_frame1, "test_frame", if_exists="fail") - assert pandasSQL.has_table("test_frame") + pandasSQL.to_sql(test_frame1, table_uuid, if_exists="fail") + assert pandasSQL.has_table(table_uuid) - msg = "Table 'test_frame' already exists" + msg = f"Table '{table_uuid}' already exists" with pytest.raises(ValueError, match=msg): - pandasSQL.to_sql(test_frame1, "test_frame", if_exists="fail") + pandasSQL.to_sql(test_frame1, table_uuid, if_exists="fail") + +@pytest.mark.parametrize( + "iris_connect_and_per_test_id", setup(all_connectable_iris), indirect=True +) +def test_read_iris_query(iris_connect_and_per_test_id, request): + conn = iris_connect_and_per_test_id["conn"] + iris_uuid = iris_connect_and_per_test_id["iris_table_uuid"] -@pytest.mark.parametrize("conn", all_connectable_iris) -def test_read_iris_query(conn, request): - conn = request.getfixturevalue(conn) - iris_frame = read_sql_query("SELECT * FROM iris", conn) + iris_frame = read_sql_query(f"SELECT * FROM {iris_uuid}", conn) check_iris_frame(iris_frame) - iris_frame = pd.read_sql("SELECT * FROM iris", conn) + iris_frame = pd.read_sql(f"SELECT * FROM {iris_uuid}", conn) check_iris_frame(iris_frame) - iris_frame = pd.read_sql("SELECT * FROM iris where 0=1", conn) + iris_frame = pd.read_sql(f"SELECT * FROM {iris_uuid} where 0=1", conn) assert iris_frame.shape == (0, 5) assert "SepalWidth" in iris_frame.columns -@pytest.mark.parametrize("conn", all_connectable_iris) -def test_read_iris_query_chunksize(conn, request): - if "adbc" in conn: +@pytest.mark.parametrize( + "iris_connect_and_per_test_id", setup(all_connectable_iris), indirect=True +) +def test_read_iris_query_chunksize(iris_connect_and_per_test_id, request): + conn = iris_connect_and_per_test_id["conn"] + iris_uuid = iris_connect_and_per_test_id["iris_table_uuid"] + conn_name = iris_connect_and_per_test_id["conn_name"] + + if "adbc" in conn_name: request.node.add_marker( pytest.mark.xfail( reason="'chunksize' not implemented for ADBC drivers", strict=True, ) ) - conn = request.getfixturevalue(conn) - iris_frame = concat(read_sql_query("SELECT * FROM iris", conn, chunksize=7)) + iris_frame = concat( + read_sql_query(f"SELECT * FROM {iris_uuid}", conn, chunksize=7) + ) check_iris_frame(iris_frame) - iris_frame = concat(pd.read_sql("SELECT * FROM iris", conn, chunksize=7)) + iris_frame = concat(pd.read_sql(f"SELECT * FROM {iris_uuid}", conn, chunksize=7)) check_iris_frame(iris_frame) - iris_frame = concat(pd.read_sql("SELECT * FROM iris where 0=1", conn, chunksize=7)) + iris_frame = concat( + pd.read_sql(f"SELECT * FROM {iris_uuid} where 0=1", conn, chunksize=7) + ) assert iris_frame.shape == (0, 5) assert "SepalWidth" in iris_frame.columns -@pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) -def test_read_iris_query_expression_with_parameter(conn, request): - if "adbc" in conn: +@pytest.mark.parametrize( + "iris_connect_and_per_test_id", setup(sqlalchemy_connectable_iris), indirect=True +) +def test_read_iris_query_expression_with_parameter( + iris_connect_and_per_test_id, request +): + conn = iris_connect_and_per_test_id["conn"] + iris_uuid = iris_connect_and_per_test_id["iris_table_uuid"] + conn_name = iris_connect_and_per_test_id["conn_name"] + + if "adbc" in conn_name: request.node.add_marker( pytest.mark.xfail( reason="'chunksize' not implemented for ADBC drivers", strict=True, ) ) - conn = request.getfixturevalue(conn) from sqlalchemy import ( MetaData, Table, @@ -1148,7 +1462,7 @@ def test_read_iris_query_expression_with_parameter(conn, request): metadata = MetaData() autoload_con = create_engine(conn) if isinstance(conn, str) else conn - iris = Table("iris", metadata, autoload_with=autoload_con) + iris = Table(iris_uuid, metadata, autoload_with=autoload_con) iris_frame = read_sql_query( select(iris), conn, params={"name": "Iris-setosa", "length": 5.1} ) @@ -1157,52 +1471,79 @@ def test_read_iris_query_expression_with_parameter(conn, request): autoload_con.dispose() -@pytest.mark.parametrize("conn", all_connectable_iris) -def test_read_iris_query_string_with_parameter(conn, request, sql_strings): - if "adbc" in conn: +@pytest.mark.parametrize( + "iris_connect_and_per_test_id", setup(all_connectable_iris), indirect=True +) +def test_read_iris_query_string_with_parameter( + iris_connect_and_per_test_id, request, sql_strings +): + conn = iris_connect_and_per_test_id["conn"] + iris_uuid = iris_connect_and_per_test_id["iris_table_uuid"] + conn_name = iris_connect_and_per_test_id["conn_name"] + + if "adbc" in conn_name: request.node.add_marker( pytest.mark.xfail( reason="'chunksize' not implemented for ADBC drivers", strict=True, ) ) - - for db, query in sql_strings["read_parameters"].items(): - if db in conn: + iris_sql_strings = sql_strings["read_parameters"] + for key, value in iris_sql_strings.items(): + iris_sql_strings[key] = iris_sql_strings[key].replace("iris", iris_uuid) + for db, query in iris_sql_strings.items(): + if db in conn_name: break else: - raise KeyError(f"No part of {conn} found in sql_strings['read_parameters']") - conn = request.getfixturevalue(conn) + raise KeyError( + f"No part of {conn_name} found in sql_strings['read_parameters']" + ) + iris_frame = read_sql_query(query, conn, params=("Iris-setosa", 5.1)) check_iris_frame(iris_frame) -@pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) -def test_read_iris_table(conn, request): +@pytest.mark.parametrize( + "iris_connect_and_per_test_id", setup(sqlalchemy_connectable_iris), indirect=True +) +def test_read_iris_table(iris_connect_and_per_test_id, request): # GH 51015 if conn = sqlite_iris_str - conn = request.getfixturevalue(conn) - iris_frame = read_sql_table("iris", conn) + conn = iris_connect_and_per_test_id["conn"] + iris_uuid = iris_connect_and_per_test_id["iris_table_uuid"] + + iris_frame = read_sql_table(iris_uuid, conn) check_iris_frame(iris_frame) - iris_frame = pd.read_sql("iris", conn) + iris_frame = pd.read_sql(iris_uuid, conn) check_iris_frame(iris_frame) -@pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) -def test_read_iris_table_chunksize(conn, request): - if "adbc" in conn: +@pytest.mark.parametrize( + "iris_connect_and_per_test_id", setup(sqlalchemy_connectable_iris), indirect=True +) +def test_read_iris_table_chunksize(iris_connect_and_per_test_id, request): + conn = iris_connect_and_per_test_id["conn"] + iris_uuid = iris_connect_and_per_test_id["iris_table_uuid"] + conn_name = iris_connect_and_per_test_id["conn_name"] + if "adbc" in conn_name: request.node.add_marker( - pytest.mark.xfail(reason="chunksize argument NotImplemented with ADBC") + pytest.mark.xfail( + reason="chunksize argument NotImplemented with ADBC", strict=True + ) ) - conn = request.getfixturevalue(conn) - iris_frame = concat(read_sql_table("iris", conn, chunksize=7)) + iris_frame = concat(read_sql_table(iris_uuid, conn, chunksize=7)) check_iris_frame(iris_frame) - iris_frame = concat(pd.read_sql("iris", conn, chunksize=7)) + iris_frame = concat(pd.read_sql(iris_uuid, conn, chunksize=7)) check_iris_frame(iris_frame) -@pytest.mark.parametrize("conn", sqlalchemy_connectable) -def test_to_sql_callable(conn, test_frame1, request): - conn = request.getfixturevalue(conn) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(sqlalchemy_connectable, table_uuid="test_to_sql_callable"), + indirect=True, +) +def test_to_sql_callable(connect_and_uuid, test_frame1, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] check = [] # used to double check function below is really being used @@ -1212,15 +1553,21 @@ def sample(pd_table, conn, keys, data_iter): conn.execute(pd_table.table.insert(), data) with pandasSQL_builder(conn, need_transaction=True) as pandasSQL: - pandasSQL.to_sql(test_frame1, "test_frame", method=sample) - assert pandasSQL.has_table("test_frame") + pandasSQL.to_sql(test_frame1, table_uuid, method=sample) + assert pandasSQL.has_table(table_uuid) assert check == [1] - assert count_rows(conn, "test_frame") == len(test_frame1) + assert count_rows(conn, table_uuid) == len(test_frame1) -@pytest.mark.parametrize("conn", all_connectable_types) -def test_default_type_conversion(conn, request): - conn_name = conn +@pytest.mark.parametrize( + "connect_and_uuid_types", + setup(sqlalchemy_connectable_types, table_uuid="test_default_type_conversion"), + indirect=True, +) +def test_default_type_conversion(connect_and_uuid_types, request): + conn = connect_and_uuid_types["conn"] + table_uuid = connect_and_uuid_types["table_uuid"] + conn_name = connect_and_uuid_types["conn_name"] if conn_name == "sqlite_buildin_types": request.applymarker( pytest.mark.xfail( @@ -1228,8 +1575,7 @@ def test_default_type_conversion(conn, request): ) ) - conn = request.getfixturevalue(conn) - df = sql.read_sql_table("types", conn) + df = sql.read_sql_table(table_uuid, conn) assert issubclass(df.FloatCol.dtype.type, np.floating) assert issubclass(df.IntCol.dtype.type, np.integer) @@ -1250,9 +1596,14 @@ def test_default_type_conversion(conn, request): assert issubclass(df.BoolColWithNull.dtype.type, np.floating) -@pytest.mark.parametrize("conn", mysql_connectable) -def test_read_procedure(conn, request): - conn = request.getfixturevalue(conn) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(mysql_connectable, table_uuid="test_read_procedure"), + indirect=True, +) +def test_read_procedure(connect_and_uuid, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] # GH 7324 # Although it is more an api test, it is added to the @@ -1261,14 +1612,14 @@ def test_read_procedure(conn, request): from sqlalchemy.engine import Engine df = DataFrame({"a": [1, 2, 3], "b": [0.1, 0.2, 0.3]}) - df.to_sql(name="test_frame", con=conn, index=False) + df.to_sql(name=table_uuid, con=conn, index=False) - proc = """DROP PROCEDURE IF EXISTS get_testdb; + proc = f"""DROP PROCEDURE IF EXISTS get_testdb; CREATE PROCEDURE get_testdb () BEGIN - SELECT * FROM test_frame; + SELECT * FROM {table_uuid}; END""" proc = text(proc) if isinstance(conn, Engine): @@ -1287,12 +1638,23 @@ def test_read_procedure(conn, request): tm.assert_frame_equal(df, res2) -@pytest.mark.parametrize("conn", postgresql_connectable) +@pytest.mark.parametrize( + "connect_and_uuid", + setup( + postgresql_connectable, table_uuid="test_copy_from_callable_insertion_method" + ), + indirect=True, +) @pytest.mark.parametrize("expected_count", [2, "Success!"]) -def test_copy_from_callable_insertion_method(conn, expected_count, request): +def test_copy_from_callable_insertion_method( + connect_and_uuid, expected_count, request +): # GH 8953 # Example in io.rst found under _io.sql.method # not available in sqlite, mysql + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + def psql_insert_copy(table, conn, keys, data_iter): # gets a DBAPI connection that can provide a cursor dbapi_conn = conn.connection @@ -1312,24 +1674,31 @@ def psql_insert_copy(table, conn, keys, data_iter): cur.copy_expert(sql=sql_query, file=s_buf) return expected_count - conn = request.getfixturevalue(conn) expected = DataFrame({"col1": [1, 2], "col2": [0.1, 0.2], "col3": ["a", "n"]}) result_count = expected.to_sql( - name="test_frame", con=conn, index=False, method=psql_insert_copy + name=table_uuid, con=conn, index=False, method=psql_insert_copy ) # GH 46891 if expected_count is None: assert result_count is None else: assert result_count == expected_count - result = sql.read_sql_table("test_frame", conn) + result = sql.read_sql_table(table_uuid, conn) tm.assert_frame_equal(result, expected) -@pytest.mark.parametrize("conn", postgresql_connectable) -def test_insertion_method_on_conflict_do_nothing(conn, request): +@pytest.mark.parametrize( + "connect_and_uuid", + setup( + postgresql_connectable, + table_uuid="test_insertion_method_on_conflict_do_nothing", + ), + indirect=True, +) +def test_insertion_method_on_conflict_do_nothing(connect_and_uuid, request): # GH 15988: Example in to_sql docstring - conn = request.getfixturevalue(conn) + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] from sqlalchemy.dialects.postgresql import insert from sqlalchemy.engine import Engine @@ -1346,8 +1715,8 @@ def insert_on_conflict(table, conn, keys, data_iter): return result.rowcount create_sql = text( - """ - CREATE TABLE test_insert_conflict ( + f""" + CREATE TABLE {table_uuid} ( a integer PRIMARY KEY, b numeric, c text @@ -1363,55 +1732,64 @@ def insert_on_conflict(table, conn, keys, data_iter): conn.execute(create_sql) expected = DataFrame([[1, 2.1, "a"]], columns=list("abc")) - expected.to_sql( - name="test_insert_conflict", con=conn, if_exists="append", index=False - ) + expected.to_sql(name=table_uuid, con=conn, if_exists="append", index=False) df_insert = DataFrame([[1, 3.2, "b"]], columns=list("abc")) inserted = df_insert.to_sql( - name="test_insert_conflict", + name=table_uuid, con=conn, index=False, if_exists="append", method=insert_on_conflict, ) - result = sql.read_sql_table("test_insert_conflict", conn) + result = sql.read_sql_table(table_uuid, conn) tm.assert_frame_equal(result, expected) assert inserted == 0 # Cleanup with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL: - pandasSQL.drop_table("test_insert_conflict") + pandasSQL.drop_table(table_uuid) -@pytest.mark.parametrize("conn", all_connectable) -def test_to_sql_on_public_schema(conn, request): - if "sqlite" in conn or "mysql" in conn: +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_to_sql_on_public_schema"), + indirect=True, +) +def test_to_sql_on_public_schema(connect_and_uuid, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + + if "sqlite" in conn_name or "mysql" in conn_name: request.applymarker( pytest.mark.xfail( reason="test for public schema only specific to postgresql" ) ) - conn = request.getfixturevalue(conn) - test_data = DataFrame([[1, 2.1, "a"], [2, 3.1, "b"]], columns=list("abc")) test_data.to_sql( - name="test_public_schema", + name=table_uuid, con=conn, if_exists="append", index=False, schema="public", ) - df_out = sql.read_sql_table("test_public_schema", conn, schema="public") + df_out = sql.read_sql_table(table_uuid, conn, schema="public") tm.assert_frame_equal(test_data, df_out) -@pytest.mark.parametrize("conn", mysql_connectable) -def test_insertion_method_on_conflict_update(conn, request): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(mysql_connectable, table_uuid="test_insertion_method_on_conflict_update"), + indirect=True, +) +def test_insertion_method_on_conflict_update(connect_and_uuid, request): # GH 14553: Example in to_sql docstring - conn = request.getfixturevalue(conn) + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] from sqlalchemy.dialects.mysql import insert from sqlalchemy.engine import Engine @@ -1425,8 +1803,8 @@ def insert_on_conflict(table, conn, keys, data_iter): return result.rowcount create_sql = text( - """ - CREATE TABLE test_insert_conflict ( + f""" + CREATE TABLE {table_uuid} ( a INT PRIMARY KEY, b FLOAT, c VARCHAR(10) @@ -1442,47 +1820,55 @@ def insert_on_conflict(table, conn, keys, data_iter): conn.execute(create_sql) df = DataFrame([[1, 2.1, "a"]], columns=list("abc")) - df.to_sql(name="test_insert_conflict", con=conn, if_exists="append", index=False) + df.to_sql(name=table_uuid, con=conn, if_exists="append", index=False) expected = DataFrame([[1, 3.2, "b"]], columns=list("abc")) inserted = expected.to_sql( - name="test_insert_conflict", + name=table_uuid, con=conn, index=False, if_exists="append", method=insert_on_conflict, ) - result = sql.read_sql_table("test_insert_conflict", conn) + result = sql.read_sql_table(table_uuid, conn) tm.assert_frame_equal(result, expected) assert inserted == 2 # Cleanup with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL: - pandasSQL.drop_table("test_insert_conflict") + pandasSQL.drop_table(table_uuid) -@pytest.mark.parametrize("conn", postgresql_connectable) -def test_read_view_postgres(conn, request): +@pytest.mark.parametrize( + "connect_and_uuid_types", + setup( + postgresql_connectable, + table_uuid="test_read_view_postgres", + uuid_views="test_read_view_postgres", + ), + indirect=True, +) +def test_read_view_postgres(connect_and_uuid_types, request): # GH 52969 - conn = request.getfixturevalue(conn) from sqlalchemy.engine import Engine from sqlalchemy.sql import text - table_name = f"group_{uuid.uuid4().hex}" - view_name = f"group_view_{uuid.uuid4().hex}" + conn = connect_and_uuid_types["conn"] + table_uuid = connect_and_uuid_types["table_uuid"] + view_uuid = connect_and_uuid_types["view_uuid"] sql_stmt = text( f""" - CREATE TABLE {table_name} ( + CREATE TABLE {table_uuid} ( group_id INTEGER, name TEXT ); - INSERT INTO {table_name} VALUES + INSERT INTO {table_uuid} VALUES (1, 'name'); - CREATE VIEW {view_name} + CREATE VIEW {view_uuid} AS - SELECT * FROM {table_name}; + SELECT * FROM {table_uuid}; """ ) if isinstance(conn, Engine): @@ -1492,32 +1878,45 @@ def test_read_view_postgres(conn, request): else: with conn.begin(): conn.execute(sql_stmt) - result = read_sql_table(view_name, conn) + result = read_sql_table(view_uuid, conn) expected = DataFrame({"group_id": [1], "name": "name"}) tm.assert_frame_equal(result, expected) -def test_read_view_sqlite(sqlite_buildin): +@pytest.mark.parametrize( + "connect_and_uuid_types", + setup( + ["sqlite_buildin"], + table_uuid="test_read_view_sqlite", + uuid_views="test_read_view_sqlite", + ), + indirect=True, +) +def test_read_view_sqlite(connect_and_uuid_types): # GH 52969 - create_table = """ -CREATE TABLE groups ( + conn = connect_and_uuid_types["conn"] + table_uuid = connect_and_uuid_types["table_uuid"] + view_uuid = connect_and_uuid_types["view_uuid"] + + create_table = f""" +CREATE TABLE {table_uuid} ( group_id INTEGER, name TEXT ); """ - insert_into = """ -INSERT INTO groups VALUES + insert_into = f""" +INSERT INTO {table_uuid} VALUES (1, 'name'); """ - create_view = """ -CREATE VIEW group_view + create_view = f""" +CREATE VIEW {view_uuid} AS -SELECT * FROM groups; +SELECT * FROM {table_uuid}; """ - sqlite_buildin.execute(create_table) - sqlite_buildin.execute(insert_into) - sqlite_buildin.execute(create_view) - result = pd.read_sql("SELECT * FROM group_view", sqlite_buildin) + conn.execute(create_table) + conn.execute(insert_into) + conn.execute(create_view) + result = pd.read_sql(f"SELECT * FROM {view_uuid}", conn) expected = DataFrame({"group_id": [1], "name": "name"}) tm.assert_frame_equal(result, expected) @@ -1533,18 +1932,25 @@ def flavor(conn_name): raise ValueError(f"unsupported connection: {conn_name}") -@pytest.mark.parametrize("conn", all_connectable_iris) -def test_read_sql_iris_parameter(conn, request, sql_strings): - if "adbc" in conn: +@pytest.mark.parametrize( + "iris_connect_and_per_test_id", setup(all_connectable_iris), indirect=True +) +def test_read_sql_iris_parameter(iris_connect_and_per_test_id, request, sql_strings): + conn = iris_connect_and_per_test_id["conn"] + iris_uuid = iris_connect_and_per_test_id["iris_table_uuid"] + conn_name = iris_connect_and_per_test_id["conn_name"] + + if "adbc" in conn_name: request.node.add_marker( pytest.mark.xfail( reason="'params' not implemented for ADBC drivers", strict=True, ) ) - conn_name = conn - conn = request.getfixturevalue(conn) - query = sql_strings["read_parameters"][flavor(conn_name)] + iris_sql_strings = sql_strings["read_parameters"][flavor(conn_name)].replace( + "iris", iris_uuid + ) + query = iris_sql_strings params = ("Iris-setosa", 5.1) with pandasSQL_builder(conn) as pandasSQL: with pandasSQL.run_transaction(): @@ -1552,9 +1958,17 @@ def test_read_sql_iris_parameter(conn, request, sql_strings): check_iris_frame(iris_frame) -@pytest.mark.parametrize("conn", all_connectable_iris) -def test_read_sql_iris_named_parameter(conn, request, sql_strings): - if "adbc" in conn: +@pytest.mark.parametrize( + "iris_connect_and_per_test_id", setup(all_connectable_iris), indirect=True +) +def test_read_sql_iris_named_parameter( + iris_connect_and_per_test_id, iris_path, request, sql_strings +): + conn = iris_connect_and_per_test_id["conn"] + iris_uuid = iris_connect_and_per_test_id["iris_table_uuid"] + conn_name = iris_connect_and_per_test_id["conn_name"] + + if "adbc" in conn_name: request.node.add_marker( pytest.mark.xfail( reason="'params' not implemented for ADBC drivers", @@ -1562,9 +1976,9 @@ def test_read_sql_iris_named_parameter(conn, request, sql_strings): ) ) - conn_name = conn - conn = request.getfixturevalue(conn) query = sql_strings["read_named_parameters"][flavor(conn_name)] + query = query.replace("iris", iris_uuid) + params = {"name": "Iris-setosa", "length": 5.1} with pandasSQL_builder(conn) as pandasSQL: with pandasSQL.run_transaction(): @@ -1572,15 +1986,24 @@ def test_read_sql_iris_named_parameter(conn, request, sql_strings): check_iris_frame(iris_frame) -@pytest.mark.parametrize("conn", all_connectable_iris) -def test_read_sql_iris_no_parameter_with_percent(conn, request, sql_strings): - if "mysql" in conn or ("postgresql" in conn and "adbc" not in conn): - request.applymarker(pytest.mark.xfail(reason="broken test")) +@pytest.mark.parametrize( + "iris_connect_and_per_test_id", setup(all_connectable_iris), indirect=True +) +def test_read_sql_iris_no_parameter_with_percent( + iris_connect_and_per_test_id, iris_path, request, sql_strings +): + conn = iris_connect_and_per_test_id["conn"] + iris_uuid = iris_connect_and_per_test_id["iris_table_uuid"] + conn_name = iris_connect_and_per_test_id["conn_name"] - conn_name = conn - conn = request.getfixturevalue(conn) + if "mysql" in conn_name or ( + "postgresql" in conn_name and "adbc" not in conn_name + ): + request.applymarker(pytest.mark.xfail(reason="broken test")) query = sql_strings["read_no_parameters_with_percent"][flavor(conn_name)] + query = query.replace("iris", iris_uuid) + with pandasSQL_builder(conn) as pandasSQL: with pandasSQL.run_transaction(): iris_frame = pandasSQL.read_query(query, params=None) @@ -1591,125 +2014,171 @@ def test_read_sql_iris_no_parameter_with_percent(conn, request, sql_strings): # -- Testing the public API -@pytest.mark.parametrize("conn", all_connectable_iris) -def test_api_read_sql_view(conn, request): - conn = request.getfixturevalue(conn) - iris_frame = sql.read_sql_query("SELECT * FROM iris_view", conn) +@pytest.mark.parametrize( + "iris_connect_and_per_test_id", setup(all_connectable_iris), indirect=True +) +def test_api_read_sql_view(iris_connect_and_per_test_id, request): + conn = iris_connect_and_per_test_id["conn"] + iris_view_uuid = iris_connect_and_per_test_id["iris_view_uuid"] + + iris_frame = sql.read_sql_query(f"SELECT * FROM {iris_view_uuid}", conn) check_iris_frame(iris_frame) -@pytest.mark.parametrize("conn", all_connectable_iris) -def test_api_read_sql_with_chunksize_no_result(conn, request): - if "adbc" in conn: +@pytest.mark.parametrize( + "iris_connect_and_per_test_id", setup(all_connectable_iris), indirect=True +) +def test_api_read_sql_with_chunksize_no_result( + iris_connect_and_per_test_id, request +): + conn = iris_connect_and_per_test_id["conn"] + iris_view_uuid = iris_connect_and_per_test_id["iris_view_uuid"] + conn_name = iris_connect_and_per_test_id["conn_name"] + if "adbc" in conn_name: request.node.add_marker( pytest.mark.xfail(reason="chunksize argument NotImplemented with ADBC") ) - conn = request.getfixturevalue(conn) - query = 'SELECT * FROM iris_view WHERE "SepalLength" < 0.0' + query = f'SELECT * FROM {iris_view_uuid} WHERE "SepalLength" < 0.0' with_batch = sql.read_sql_query(query, conn, chunksize=5) without_batch = sql.read_sql_query(query, conn) tm.assert_frame_equal(concat(with_batch), without_batch) -@pytest.mark.parametrize("conn", all_connectable) -def test_api_to_sql(conn, request, test_frame1): - conn = request.getfixturevalue(conn) - if sql.has_table("test_frame1", conn): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_api_to_sql"), + indirect=True, +) +def test_api_to_sql(connect_and_uuid, request, test_frame1): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + if sql.has_table(table_uuid, conn): with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL: - pandasSQL.drop_table("test_frame1") + pandasSQL.drop_table(table_uuid) - sql.to_sql(test_frame1, "test_frame1", conn) - assert sql.has_table("test_frame1", conn) + sql.to_sql(test_frame1, table_uuid, conn) + assert sql.has_table(table_uuid, conn) -@pytest.mark.parametrize("conn", all_connectable) -def test_api_to_sql_fail(conn, request, test_frame1): - conn = request.getfixturevalue(conn) - if sql.has_table("test_frame2", conn): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_api_to_sql_fail"), + indirect=True, +) +def test_api_to_sql_fail(connect_and_uuid, request, test_frame1): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + if sql.has_table(table_uuid, conn): with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL: - pandasSQL.drop_table("test_frame2") + pandasSQL.drop_table(table_uuid) - sql.to_sql(test_frame1, "test_frame2", conn, if_exists="fail") - assert sql.has_table("test_frame2", conn) + sql.to_sql(test_frame1, table_uuid, conn, if_exists="fail") + assert sql.has_table(table_uuid, conn) - msg = "Table 'test_frame2' already exists" + msg = f"Table '{table_uuid}' already exists" with pytest.raises(ValueError, match=msg): - sql.to_sql(test_frame1, "test_frame2", conn, if_exists="fail") + sql.to_sql(test_frame1, table_uuid, conn, if_exists="fail") -@pytest.mark.parametrize("conn", all_connectable) -def test_api_to_sql_replace(conn, request, test_frame1): - conn = request.getfixturevalue(conn) - if sql.has_table("test_frame3", conn): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_api_to_sql_replace"), + indirect=True, +) +def test_api_to_sql_replace(connect_and_uuid, request, test_frame1): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + if sql.has_table(table_uuid, conn): with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL: - pandasSQL.drop_table("test_frame3") + pandasSQL.drop_table(table_uuid) - sql.to_sql(test_frame1, "test_frame3", conn, if_exists="fail") + sql.to_sql(test_frame1, table_uuid, conn, if_exists="fail") # Add to table again - sql.to_sql(test_frame1, "test_frame3", conn, if_exists="replace") - assert sql.has_table("test_frame3", conn) + sql.to_sql(test_frame1, table_uuid, conn, if_exists="replace") + assert sql.has_table(table_uuid, conn) num_entries = len(test_frame1) - num_rows = count_rows(conn, "test_frame3") + num_rows = count_rows(conn, table_uuid) assert num_rows == num_entries -@pytest.mark.parametrize("conn", all_connectable) -def test_api_to_sql_append(conn, request, test_frame1): - conn = request.getfixturevalue(conn) - if sql.has_table("test_frame4", conn): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_api_to_sql_append"), + indirect=True, +) +def test_api_to_sql_append(connect_and_uuid, request, test_frame1): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + if sql.has_table(table_uuid, conn): with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL: - pandasSQL.drop_table("test_frame4") + pandasSQL.drop_table(table_uuid) - assert sql.to_sql(test_frame1, "test_frame4", conn, if_exists="fail") == 4 + assert sql.to_sql(test_frame1, table_uuid, conn, if_exists="fail") == 4 # Add to table again - assert sql.to_sql(test_frame1, "test_frame4", conn, if_exists="append") == 4 - assert sql.has_table("test_frame4", conn) + assert sql.to_sql(test_frame1, table_uuid, conn, if_exists="append") == 4 + assert sql.has_table(table_uuid, conn) num_entries = 2 * len(test_frame1) - num_rows = count_rows(conn, "test_frame4") + num_rows = count_rows(conn, table_uuid) assert num_rows == num_entries -@pytest.mark.parametrize("conn", all_connectable) -def test_api_to_sql_type_mapping(conn, request, test_frame3): - conn = request.getfixturevalue(conn) - if sql.has_table("test_frame5", conn): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_api_to_sql_type_mapping"), + indirect=True, +) +def test_api_to_sql_type_mapping(connect_and_uuid, request, test_frame3): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + if sql.has_table(table_uuid, conn): with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL: - pandasSQL.drop_table("test_frame5") + pandasSQL.drop_table(table_uuid) - sql.to_sql(test_frame3, "test_frame5", conn, index=False) - result = sql.read_sql("SELECT * FROM test_frame5", conn) + sql.to_sql(test_frame3, table_uuid, conn, index=False) + result = sql.read_sql(f"SELECT * FROM {table_uuid}", conn) tm.assert_frame_equal(test_frame3, result) -@pytest.mark.parametrize("conn", all_connectable) -def test_api_to_sql_series(conn, request): - conn = request.getfixturevalue(conn) - if sql.has_table("test_series", conn): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_api_to_sql_series"), + indirect=True, +) +def test_api_to_sql_series(connect_and_uuid, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + if sql.has_table(table_uuid, conn): with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL: - pandasSQL.drop_table("test_series") + pandasSQL.drop_table(table_uuid) s = Series(np.arange(5, dtype="int64"), name="series") - sql.to_sql(s, "test_series", conn, index=False) - s2 = sql.read_sql_query("SELECT * FROM test_series", conn) + sql.to_sql(s, table_uuid, conn, index=False) + s2 = sql.read_sql_query(f"SELECT * FROM {table_uuid}", conn) tm.assert_frame_equal(s.to_frame(), s2) -@pytest.mark.parametrize("conn", all_connectable) -def test_api_roundtrip(conn, request, test_frame1): - conn_name = conn - conn = request.getfixturevalue(conn) - if sql.has_table("test_frame_roundtrip", conn): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_api_roundtrip"), + indirect=True, +) +def test_api_roundtrip(connect_and_uuid, request, test_frame1): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + + if sql.has_table(table_uuid, conn): with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL: - pandasSQL.drop_table("test_frame_roundtrip") + pandasSQL.drop_table(table_uuid) - sql.to_sql(test_frame1, "test_frame_roundtrip", con=conn) - result = sql.read_sql_query("SELECT * FROM test_frame_roundtrip", con=conn) + sql.to_sql(test_frame1, table_uuid, con=conn) + result = sql.read_sql_query(f"SELECT * FROM {table_uuid}", con=conn) # HACK! if "adbc" in conn_name: @@ -1719,50 +2188,69 @@ def test_api_roundtrip(conn, request, test_frame1): tm.assert_frame_equal(result, test_frame1) -@pytest.mark.parametrize("conn", all_connectable) -def test_api_roundtrip_chunksize(conn, request, test_frame1): - if "adbc" in conn: +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_api_roundtrip_chunksize"), + indirect=True, +) +def test_api_roundtrip_chunksize(connect_and_uuid, request, test_frame1): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + + if "adbc" in conn_name: request.node.add_marker( pytest.mark.xfail(reason="chunksize argument NotImplemented with ADBC") ) - conn = request.getfixturevalue(conn) - if sql.has_table("test_frame_roundtrip", conn): + + if sql.has_table(table_uuid, conn): with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL: - pandasSQL.drop_table("test_frame_roundtrip") + pandasSQL.drop_table(table_uuid) sql.to_sql( test_frame1, - "test_frame_roundtrip", + table_uuid, con=conn, index=False, chunksize=2, ) - result = sql.read_sql_query("SELECT * FROM test_frame_roundtrip", con=conn) + result = sql.read_sql_query(f"SELECT * FROM {table_uuid}", con=conn) tm.assert_frame_equal(result, test_frame1) -@pytest.mark.parametrize("conn", all_connectable_iris) -def test_api_execute_sql(conn, request): +@pytest.mark.parametrize( + "iris_connect_and_per_test_id", setup(all_connectable_iris), indirect=True +) +def test_api_execute_sql(iris_connect_and_per_test_id, request): # drop_sql = "DROP TABLE IF EXISTS test" # should already be done - conn = request.getfixturevalue(conn) + conn = iris_connect_and_per_test_id["conn"] + iris_uuid = iris_connect_and_per_test_id["iris_table_uuid"] + with sql.pandasSQL_builder(conn) as pandas_sql: - iris_results = pandas_sql.execute("SELECT * FROM iris") + iris_results = pandas_sql.execute(f"SELECT * FROM {iris_uuid}") row = iris_results.fetchone() iris_results.close() assert list(row) == [5.1, 3.5, 1.4, 0.2, "Iris-setosa"] -@pytest.mark.parametrize("conn", all_connectable_types) -def test_api_date_parsing(conn, request): - conn_name = conn - conn = request.getfixturevalue(conn) +@pytest.mark.parametrize( + "connect_and_uuid_types", + setup(all_connectable_types, table_uuid="test_api_date_parsing"), + indirect=True, +) +def test_api_date_parsing(connect_and_uuid_types, request): + conn = connect_and_uuid_types["conn"] + table_uuid = connect_and_uuid_types["table_uuid"] + conn_name = connect_and_uuid_types["conn_name"] # Test date parsing in read_sql # No Parsing - df = sql.read_sql_query("SELECT * FROM types", conn) + df = sql.read_sql_query(f"SELECT * FROM {table_uuid}", conn) if not ("mysql" in conn_name or "postgres" in conn_name): assert not issubclass(df.DateCol.dtype.type, np.datetime64) - df = sql.read_sql_query("SELECT * FROM types", conn, parse_dates=["DateCol"]) + df = sql.read_sql_query( + f"SELECT * FROM {table_uuid}", conn, parse_dates=["DateCol"] + ) assert issubclass(df.DateCol.dtype.type, np.datetime64) assert df.DateCol.tolist() == [ Timestamp(2000, 1, 3, 0, 0, 0), @@ -1770,7 +2258,7 @@ def test_api_date_parsing(conn, request): ] df = sql.read_sql_query( - "SELECT * FROM types", + f"SELECT * FROM {table_uuid}", conn, parse_dates={"DateCol": "%Y-%m-%d %H:%M:%S"}, ) @@ -1780,7 +2268,9 @@ def test_api_date_parsing(conn, request): Timestamp(2000, 1, 4, 0, 0, 0), ] - df = sql.read_sql_query("SELECT * FROM types", conn, parse_dates=["IntDateCol"]) + df = sql.read_sql_query( + f"SELECT * FROM {table_uuid}", conn, parse_dates=["IntDateCol"] + ) assert issubclass(df.IntDateCol.dtype.type, np.datetime64) assert df.IntDateCol.tolist() == [ Timestamp(1986, 12, 25, 0, 0, 0), @@ -1788,7 +2278,7 @@ def test_api_date_parsing(conn, request): ] df = sql.read_sql_query( - "SELECT * FROM types", conn, parse_dates={"IntDateCol": "s"} + f"SELECT * FROM {table_uuid}", conn, parse_dates={"IntDateCol": "s"} ) assert issubclass(df.IntDateCol.dtype.type, np.datetime64) assert df.IntDateCol.tolist() == [ @@ -1797,7 +2287,7 @@ def test_api_date_parsing(conn, request): ] df = sql.read_sql_query( - "SELECT * FROM types", + f"SELECT * FROM {table_uuid}", conn, parse_dates={"IntDateOnlyCol": "%Y%m%d"}, ) @@ -1808,7 +2298,11 @@ def test_api_date_parsing(conn, request): ] -@pytest.mark.parametrize("conn", all_connectable_types) +@pytest.mark.parametrize( + "connect_and_uuid_types", + setup(all_connectable_types, table_uuid="test_api_custom_dateparsing_error"), + indirect=True, +) @pytest.mark.parametrize("error", ["raise", "coerce"]) @pytest.mark.parametrize( "read_sql, text, mode", @@ -1824,14 +2318,16 @@ def test_api_date_parsing(conn, request): ], ) def test_api_custom_dateparsing_error( - conn, request, read_sql, text, mode, error, types_data_frame + connect_and_uuid_types, request, read_sql, text, mode, error, types_data_frame ): - conn_name = conn - conn = request.getfixturevalue(conn) + conn = connect_and_uuid_types["conn"] + table_uuid = connect_and_uuid_types["table_uuid"] + conn_name = connect_and_uuid_types["conn_name"] if text == "types" and conn_name == "sqlite_buildin_types": request.applymarker( pytest.mark.xfail(reason="failing combination of arguments") ) + text = text.replace("types", table_uuid) expected = types_data_frame.astype({"DateCol": "datetime64[s]"}) @@ -1865,12 +2361,17 @@ def test_api_custom_dateparsing_error( tm.assert_frame_equal(result, expected) -@pytest.mark.parametrize("conn", all_connectable_types) -def test_api_date_and_index(conn, request): +@pytest.mark.parametrize( + "connect_and_uuid_types", + setup(all_connectable_types, table_uuid="test_api_date_and_index"), + indirect=True, +) +def test_api_date_and_index(connect_and_uuid_types, request): # Test case where same column appears in parse_date and index_col - conn = request.getfixturevalue(conn) + conn = connect_and_uuid_types["conn"] + table_uuid = connect_and_uuid_types["table_uuid"] df = sql.read_sql_query( - "SELECT * FROM types", + f"SELECT * FROM {table_uuid}", conn, index_col="DateCol", parse_dates=["DateCol", "IntDateCol"], @@ -1880,14 +2381,20 @@ def test_api_date_and_index(conn, request): assert issubclass(df.IntDateCol.dtype.type, np.datetime64) -@pytest.mark.parametrize("conn", all_connectable) -def test_api_timedelta(conn, request): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_api_timedelta"), + indirect=True, +) +def test_api_timedelta(connect_and_uuid, request): # see #6921 - conn_name = conn - conn = request.getfixturevalue(conn) - if sql.has_table("test_timedelta", conn): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + + if sql.has_table(table_uuid, conn): with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL: - pandasSQL.drop_table("test_timedelta") + pandasSQL.drop_table(table_uuid) df = to_timedelta(Series(["00:00:01", "00:00:03"], name="foo")).to_frame() @@ -1907,9 +2414,9 @@ def test_api_timedelta(conn, request): exp_warning = UserWarning with tm.assert_produces_warning(exp_warning, check_stacklevel=False): - result_count = df.to_sql(name="test_timedelta", con=conn) + result_count = df.to_sql(name=table_uuid, con=conn) assert result_count == 2 - result = sql.read_sql_query("SELECT * FROM test_timedelta", conn) + result = sql.read_sql_query(f"SELECT * FROM {table_uuid}", conn) if conn_name == "postgresql_adbc_conn": # TODO: Postgres stores an INTERVAL, which ADBC reads as a Month-Day-Nano @@ -1927,10 +2434,16 @@ def test_api_timedelta(conn, request): tm.assert_series_equal(result["foo"], expected) -@pytest.mark.parametrize("conn", all_connectable) -def test_api_complex_raises(conn, request): - conn_name = conn - conn = request.getfixturevalue(conn) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_api_complex_raises"), + indirect=True, +) +def test_api_complex_raises(connect_and_uuid, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + df = DataFrame({"a": [1 + 1j, 2j]}) if "adbc" in conn_name: @@ -1938,10 +2451,14 @@ def test_api_complex_raises(conn, request): else: msg = "Complex datatypes not supported" with pytest.raises(ValueError, match=msg): - assert df.to_sql("test_complex", con=conn) is None + assert df.to_sql(table_uuid, con=conn) is None -@pytest.mark.parametrize("conn", all_connectable) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_api_to_sql_index_label"), + indirect=True, +) @pytest.mark.parametrize( "index_name,index_label,expected", [ @@ -1959,31 +2476,44 @@ def test_api_complex_raises(conn, request): (None, 0, "0"), ], ) -def test_api_to_sql_index_label(conn, request, index_name, index_label, expected): - if "adbc" in conn: +def test_api_to_sql_index_label( + connect_and_uuid, request, index_name, index_label, expected +): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + + if "adbc" in conn_name: request.node.add_marker( pytest.mark.xfail(reason="index_label argument NotImplemented with ADBC") ) - conn = request.getfixturevalue(conn) - if sql.has_table("test_index_label", conn): + if sql.has_table(table_uuid, conn): with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL: - pandasSQL.drop_table("test_index_label") + pandasSQL.drop_table(table_uuid) temp_frame = DataFrame({"col1": range(4)}) temp_frame.index.name = index_name - query = "SELECT * FROM test_index_label" - sql.to_sql(temp_frame, "test_index_label", conn, index_label=index_label) + query = f"SELECT * FROM {table_uuid}" + sql.to_sql(temp_frame, table_uuid, conn, index_label=index_label) frame = sql.read_sql_query(query, conn) assert frame.columns[0] == expected -@pytest.mark.parametrize("conn", all_connectable) -def test_api_to_sql_index_label_multiindex(conn, request): - conn_name = conn +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_api_to_sql_index_label_multiindex"), + indirect=True, +) +def test_api_to_sql_index_label_multiindex(connect_and_uuid, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + if "mysql" in conn_name: request.applymarker( pytest.mark.xfail( - reason="MySQL can fail using TEXT without length as key", strict=False + reason="MySQL can fail using TEXT without length as key", + strict=False, ) ) elif "adbc" in conn_name: @@ -1991,10 +2521,9 @@ def test_api_to_sql_index_label_multiindex(conn, request): pytest.mark.xfail(reason="index_label argument NotImplemented with ADBC") ) - conn = request.getfixturevalue(conn) - if sql.has_table("test_index_label", conn): + if sql.has_table(table_uuid, conn): with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL: - pandasSQL.drop_table("test_index_label") + pandasSQL.drop_table(table_uuid) expected_row_count = 4 temp_frame = DataFrame( @@ -2003,60 +2532,66 @@ def test_api_to_sql_index_label_multiindex(conn, request): ) # no index name, defaults to 'level_0' and 'level_1' - result = sql.to_sql(temp_frame, "test_index_label", conn) + result = sql.to_sql(temp_frame, table_uuid, conn) assert result == expected_row_count - frame = sql.read_sql_query("SELECT * FROM test_index_label", conn) + frame = sql.read_sql_query(f"SELECT * FROM {table_uuid}", conn) assert frame.columns[0] == "level_0" assert frame.columns[1] == "level_1" # specifying index_label result = sql.to_sql( temp_frame, - "test_index_label", + table_uuid, conn, if_exists="replace", index_label=["A", "B"], ) assert result == expected_row_count - frame = sql.read_sql_query("SELECT * FROM test_index_label", conn) + frame = sql.read_sql_query(f"SELECT * FROM {table_uuid}", conn) assert frame.columns[:2].tolist() == ["A", "B"] # using the index name temp_frame.index.names = ["A", "B"] - result = sql.to_sql(temp_frame, "test_index_label", conn, if_exists="replace") + result = sql.to_sql(temp_frame, table_uuid, conn, if_exists="replace") assert result == expected_row_count - frame = sql.read_sql_query("SELECT * FROM test_index_label", conn) + frame = sql.read_sql_query(f"SELECT * FROM {table_uuid}", conn) assert frame.columns[:2].tolist() == ["A", "B"] # has index name, but specifying index_label result = sql.to_sql( temp_frame, - "test_index_label", + table_uuid, conn, if_exists="replace", index_label=["C", "D"], ) assert result == expected_row_count - frame = sql.read_sql_query("SELECT * FROM test_index_label", conn) + frame = sql.read_sql_query(f"SELECT * FROM {table_uuid}", conn) assert frame.columns[:2].tolist() == ["C", "D"] msg = "Length of 'index_label' should match number of levels, which is 2" with pytest.raises(ValueError, match=msg): sql.to_sql( temp_frame, - "test_index_label", + table_uuid, conn, if_exists="replace", index_label="C", ) -@pytest.mark.parametrize("conn", all_connectable) -def test_api_multiindex_roundtrip(conn, request): - conn = request.getfixturevalue(conn) - if sql.has_table("test_multiindex_roundtrip", conn): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_api_multiindex_roundtrip"), + indirect=True, +) +def test_api_multiindex_roundtrip(connect_and_uuid, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + + if sql.has_table(table_uuid, conn): with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL: - pandasSQL.drop_table("test_multiindex_roundtrip") + pandasSQL.drop_table(table_uuid) df = DataFrame.from_records( [(1, 2.1, "line1"), (2, 1.5, "line2")], @@ -2064,14 +2599,18 @@ def test_api_multiindex_roundtrip(conn, request): index=["A", "B"], ) - df.to_sql(name="test_multiindex_roundtrip", con=conn) + df.to_sql(name=table_uuid, con=conn) result = sql.read_sql_query( - "SELECT * FROM test_multiindex_roundtrip", conn, index_col=["A", "B"] + f"SELECT * FROM {table_uuid}", conn, index_col=["A", "B"] ) tm.assert_frame_equal(df, result, check_index_type=True) -@pytest.mark.parametrize("conn", all_connectable) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_api_dtype_argument"), + indirect=True, +) @pytest.mark.parametrize( "dtype", [ @@ -2081,75 +2620,102 @@ def test_api_multiindex_roundtrip(conn, request): {"A": int, "B": float}, ], ) -def test_api_dtype_argument(conn, request, dtype): +def test_api_dtype_argument(connect_and_uuid, request, dtype): # GH10285 Add dtype argument to read_sql_query - conn_name = conn - conn = request.getfixturevalue(conn) - if sql.has_table("test_dtype_argument", conn): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + + if sql.has_table(table_uuid, conn): with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL: - pandasSQL.drop_table("test_dtype_argument") + pandasSQL.drop_table(table_uuid) df = DataFrame([[1.2, 3.4], [5.6, 7.8]], columns=["A", "B"]) - assert df.to_sql(name="test_dtype_argument", con=conn) == 2 + assert df.to_sql(name=table_uuid, con=conn) == 2 expected = df.astype(dtype) if "postgres" in conn_name: - query = 'SELECT "A", "B" FROM test_dtype_argument' + query = f'SELECT "A", "B" FROM {table_uuid}' else: - query = "SELECT A, B FROM test_dtype_argument" + query = f"SELECT A, B FROM {table_uuid}" result = sql.read_sql_query(query, con=conn, dtype=dtype) tm.assert_frame_equal(result, expected) -@pytest.mark.parametrize("conn", all_connectable) -def test_api_integer_col_names(conn, request): - conn = request.getfixturevalue(conn) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_api_integer_col_names"), + indirect=True, +) +def test_api_integer_col_names(connect_and_uuid, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] df = DataFrame([[1, 2], [3, 4]], columns=[0, 1]) - sql.to_sql(df, "test_frame_integer_col_names", conn, if_exists="replace") + sql.to_sql(df, table_uuid, conn, if_exists="replace") + +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_api_get_schema"), + indirect=True, +) +def test_api_get_schema(connect_and_uuid, request, test_frame1): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] -@pytest.mark.parametrize("conn", all_connectable) -def test_api_get_schema(conn, request, test_frame1): - if "adbc" in conn: + if "adbc" in conn_name: request.node.add_marker( pytest.mark.xfail( reason="'get_schema' not implemented for ADBC drivers", strict=True, ) ) - conn = request.getfixturevalue(conn) - create_sql = sql.get_schema(test_frame1, "test", con=conn) + create_sql = sql.get_schema(test_frame1, table_uuid, con=conn) assert "CREATE" in create_sql -@pytest.mark.parametrize("conn", all_connectable) -def test_api_get_schema_with_schema(conn, request, test_frame1): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_api_get_schema_with_schema"), + indirect=True, +) +def test_api_get_schema_with_schema(connect_and_uuid, request, test_frame1): # GH28486 - if "adbc" in conn: + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + + if "adbc" in conn_name: request.node.add_marker( pytest.mark.xfail( reason="'get_schema' not implemented for ADBC drivers", strict=True, ) ) - conn = request.getfixturevalue(conn) - create_sql = sql.get_schema(test_frame1, "test", con=conn, schema="pypi") + create_sql = sql.get_schema(test_frame1, table_uuid, con=conn, schema="pypi") assert "CREATE TABLE pypi." in create_sql -@pytest.mark.parametrize("conn", all_connectable) -def test_api_get_schema_dtypes(conn, request): - if "adbc" in conn: +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_api_get_schema_dtypes"), + indirect=True, +) +def test_api_get_schema_dtypes(connect_and_uuid, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + + if "adbc" in conn_name: request.node.add_marker( pytest.mark.xfail( reason="'get_schema' not implemented for ADBC drivers", strict=True, ) ) - conn_name = conn - conn = request.getfixturevalue(conn) float_frame = DataFrame({"a": [1.1, 1.2], "b": [2.1, 2.2]}) if conn_name == "sqlite_buildin": @@ -2158,66 +2724,85 @@ def test_api_get_schema_dtypes(conn, request): from sqlalchemy import Integer dtype = Integer - create_sql = sql.get_schema(float_frame, "test", con=conn, dtype={"b": dtype}) + create_sql = sql.get_schema( + float_frame, table_uuid, con=conn, dtype={"b": dtype} + ) assert "CREATE" in create_sql assert "INTEGER" in create_sql -@pytest.mark.parametrize("conn", all_connectable) -def test_api_get_schema_keys(conn, request, test_frame1): - if "adbc" in conn: +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_api_get_schema_keys"), + indirect=True, +) +def test_api_get_schema_keys(connect_and_uuid, request, test_frame1): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + + if "adbc" in conn_name: request.node.add_marker( pytest.mark.xfail( reason="'get_schema' not implemented for ADBC drivers", strict=True, ) ) - conn_name = conn - conn = request.getfixturevalue(conn) + frame = DataFrame({"Col1": [1.1, 1.2], "Col2": [2.1, 2.2]}) - create_sql = sql.get_schema(frame, "test", con=conn, keys="Col1") + + create_sql = sql.get_schema(frame, table_uuid, con=conn, keys="Col1") if "mysql" in conn_name: - constraint_sentence = "CONSTRAINT test_pk PRIMARY KEY (`Col1`)" + constraint_sentence = f"CONSTRAINT {table_uuid}_pk PRIMARY KEY (`Col1`)" else: - constraint_sentence = 'CONSTRAINT test_pk PRIMARY KEY ("Col1")' + constraint_sentence = f'CONSTRAINT {table_uuid}_pk PRIMARY KEY ("Col1")' assert constraint_sentence in create_sql # multiple columns as key (GH10385) - create_sql = sql.get_schema(test_frame1, "test", con=conn, keys=["A", "B"]) + create_sql = sql.get_schema(test_frame1, table_uuid, con=conn, keys=["A", "B"]) if "mysql" in conn_name: - constraint_sentence = "CONSTRAINT test_pk PRIMARY KEY (`A`, `B`)" + constraint_sentence = f"CONSTRAINT {table_uuid}_pk PRIMARY KEY (`A`, `B`)" else: - constraint_sentence = 'CONSTRAINT test_pk PRIMARY KEY ("A", "B")' + constraint_sentence = f'CONSTRAINT {table_uuid}_pk PRIMARY KEY ("A", "B")' assert constraint_sentence in create_sql -@pytest.mark.parametrize("conn", all_connectable) -def test_api_chunksize_read(conn, request): - if "adbc" in conn: +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_api_chunksize_read"), + indirect=True, +) +def test_api_chunksize_read(connect_and_uuid, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + + if "adbc" in conn_name: request.node.add_marker( pytest.mark.xfail(reason="chunksize argument NotImplemented with ADBC") ) - conn_name = conn - conn = request.getfixturevalue(conn) - if sql.has_table("test_chunksize", conn): + + if sql.has_table(table_uuid, conn): with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL: - pandasSQL.drop_table("test_chunksize") + pandasSQL.drop_table(table_uuid) df = DataFrame( np.random.default_rng(2).standard_normal((22, 5)), columns=list("abcde") ) - df.to_sql(name="test_chunksize", con=conn, index=False) + df.to_sql(name=table_uuid, con=conn, index=False) # reading the query in one time - res1 = sql.read_sql_query("select * from test_chunksize", conn) + res1 = sql.read_sql_query(f"select * from {table_uuid}", conn) # reading the query in chunks with read_sql_query res2 = DataFrame() i = 0 sizes = [5, 5, 5, 5, 2] - for chunk in sql.read_sql_query("select * from test_chunksize", conn, chunksize=5): + for chunk in sql.read_sql_query( + f"select * from {table_uuid}", conn, chunksize=5 + ): res2 = concat([res2, chunk], ignore_index=True) assert len(chunk) == sizes[i] i += 1 @@ -2227,13 +2812,13 @@ def test_api_chunksize_read(conn, request): # reading the query in chunks with read_sql_query if conn_name == "sqlite_buildin": with pytest.raises(NotImplementedError, match=""): - sql.read_sql_table("test_chunksize", conn, chunksize=5) + sql.read_sql_table(table_uuid, conn, chunksize=5) else: res3 = DataFrame() i = 0 sizes = [5, 5, 5, 5, 2] - for chunk in sql.read_sql_table("test_chunksize", conn, chunksize=5): + for chunk in sql.read_sql_table(table_uuid, conn, chunksize=5): res3 = concat([res3, chunk], ignore_index=True) assert len(chunk) == sizes[i] i += 1 @@ -2241,9 +2826,17 @@ def test_api_chunksize_read(conn, request): tm.assert_frame_equal(res1, res3) -@pytest.mark.parametrize("conn", all_connectable) -def test_api_categorical(conn, request): - if conn == "postgresql_adbc_conn": +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_api_categorical"), + indirect=True, +) +def test_api_categorical(connect_and_uuid, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + + if conn_name == "postgresql_adbc_conn": adbc = import_optional_dependency("adbc_driver_postgresql", errors="ignore") if adbc is not None and Version(adbc.__version__) < Version("0.9.0"): request.node.add_marker( @@ -2254,10 +2847,10 @@ def test_api_categorical(conn, request): ) # GH8624 # test that categorical gets written correctly as dense column - conn = request.getfixturevalue(conn) - if sql.has_table("test_categorical", conn): + + if sql.has_table(table_uuid, conn): with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL: - pandasSQL.drop_table("test_categorical") + pandasSQL.drop_table(table_uuid) df = DataFrame( { @@ -2268,68 +2861,87 @@ def test_api_categorical(conn, request): df2 = df.copy() df2["person_name"] = df2["person_name"].astype("category") - df2.to_sql(name="test_categorical", con=conn, index=False) - res = sql.read_sql_query("SELECT * FROM test_categorical", conn) + df2.to_sql(name=table_uuid, con=conn, index=False) + res = sql.read_sql_query(f"SELECT * FROM {table_uuid}", conn) tm.assert_frame_equal(res, df) -@pytest.mark.parametrize("conn", all_connectable) -def test_api_unicode_column_name(conn, request): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_api_unicode_column_name"), + indirect=True, +) +def test_api_unicode_column_name(connect_and_uuid, request): # GH 11431 - conn = request.getfixturevalue(conn) - if sql.has_table("test_unicode", conn): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + if sql.has_table(table_uuid, conn): with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL: - pandasSQL.drop_table("test_unicode") + pandasSQL.drop_table(table_uuid) df = DataFrame([[1, 2], [3, 4]], columns=["\xe9", "b"]) - df.to_sql(name="test_unicode", con=conn, index=False) + df.to_sql(name=table_uuid, con=conn, index=False) -@pytest.mark.parametrize("conn", all_connectable) -def test_api_escaped_table_name(conn, request): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_api_escaped_table_name"), + indirect=True, +) +def test_api_escaped_table_name(connect_and_uuid, request): # GH 13206 - conn_name = conn - conn = request.getfixturevalue(conn) - if sql.has_table("d1187b08-4943-4c8d-a7f6", conn): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + + if sql.has_table(table_uuid, conn): with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL: - pandasSQL.drop_table("d1187b08-4943-4c8d-a7f6") + pandasSQL.drop_table(table_uuid) df = DataFrame({"A": [0, 1, 2], "B": [0.2, np.nan, 5.6]}) - df.to_sql(name="d1187b08-4943-4c8d-a7f6", con=conn, index=False) + df.to_sql(name=table_uuid, con=conn, index=False) if "postgres" in conn_name: - query = 'SELECT * FROM "d1187b08-4943-4c8d-a7f6"' + query = f'SELECT * FROM "{table_uuid}"' else: - query = "SELECT * FROM `d1187b08-4943-4c8d-a7f6`" + query = f"SELECT * FROM `{table_uuid}`" res = sql.read_sql_query(query, conn) tm.assert_frame_equal(res, df) -@pytest.mark.parametrize("conn", all_connectable) -def test_api_read_sql_duplicate_columns(conn, request): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_api_read_sql_duplicate_columns"), + indirect=True, +) +def test_api_read_sql_duplicate_columns(connect_and_uuid, request): # GH#53117 - if "adbc" in conn: + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + + if "adbc" in conn_name: pa = pytest.importorskip("pyarrow") if not ( Version(pa.__version__) >= Version("16.0") - and conn in ["sqlite_adbc_conn", "postgresql_adbc_conn"] + and conn_name in ["sqlite_adbc_conn", "postgresql_adbc_conn"] ): request.node.add_marker( pytest.mark.xfail( reason="pyarrow->pandas throws ValueError", strict=True ) ) - conn = request.getfixturevalue(conn) - if sql.has_table("test_table", conn): + + if sql.has_table(table_uuid, conn): with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL: - pandasSQL.drop_table("test_table") + pandasSQL.drop_table(table_uuid) df = DataFrame({"a": [1, 2, 3], "b": [0.1, 0.2, 0.3], "c": 1}) - df.to_sql(name="test_table", con=conn, index=False) + df.to_sql(name=table_uuid, con=conn, index=False) - result = pd.read_sql("SELECT a, b, a +1 as a, c FROM test_table", conn) + result = pd.read_sql(f"SELECT a, b, a +1 as a, c FROM {table_uuid}", conn) expected = DataFrame( [[1, 0.1, 2, 1], [2, 0.2, 3, 1], [3, 0.3, 4, 1]], columns=["a", "b", "a", "c"], @@ -2337,73 +2949,99 @@ def test_api_read_sql_duplicate_columns(conn, request): tm.assert_frame_equal(result, expected) -@pytest.mark.parametrize("conn", all_connectable) -def test_read_table_columns(conn, request, test_frame1): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_read_table_columns"), + indirect=True, +) +def test_read_table_columns(connect_and_uuid, request, test_frame1): # test columns argument in read_table - conn_name = conn + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + if conn_name == "sqlite_buildin": request.applymarker(pytest.mark.xfail(reason="Not Implemented")) - conn = request.getfixturevalue(conn) - sql.to_sql(test_frame1, "test_frame", conn) + sql.to_sql(test_frame1, table_uuid, conn) cols = ["A", "B"] - result = sql.read_sql_table("test_frame", conn, columns=cols) + result = sql.read_sql_table(table_uuid, conn, columns=cols) assert result.columns.tolist() == cols -@pytest.mark.parametrize("conn", all_connectable) -def test_read_table_index_col(conn, request, test_frame1): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_read_table_index_col"), + indirect=True, +) +def test_read_table_index_col(connect_and_uuid, request, test_frame1): # test columns argument in read_table - conn_name = conn + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + if conn_name == "sqlite_buildin": request.applymarker(pytest.mark.xfail(reason="Not Implemented")) - conn = request.getfixturevalue(conn) - sql.to_sql(test_frame1, "test_frame", conn) + sql.to_sql(test_frame1, table_uuid, conn) - result = sql.read_sql_table("test_frame", conn, index_col="index") + result = sql.read_sql_table(table_uuid, conn, index_col="index") assert result.index.names == ["index"] - result = sql.read_sql_table("test_frame", conn, index_col=["A", "B"]) + result = sql.read_sql_table(table_uuid, conn, index_col=["A", "B"]) assert result.index.names == ["A", "B"] result = sql.read_sql_table( - "test_frame", conn, index_col=["A", "B"], columns=["C", "D"] + table_uuid, conn, index_col=["A", "B"], columns=["C", "D"] ) assert result.index.names == ["A", "B"] assert result.columns.tolist() == ["C", "D"] -@pytest.mark.parametrize("conn", all_connectable_iris) -def test_read_sql_delegate(conn, request): - if conn == "sqlite_buildin_iris": +@pytest.mark.parametrize( + "iris_connect_and_per_test_id", setup(all_connectable_iris), indirect=True +) +def test_read_sql_delegate(iris_connect_and_per_test_id, request): + conn = iris_connect_and_per_test_id["conn"] + iris_uuid = iris_connect_and_per_test_id["iris_table_uuid"] + conn_name = iris_connect_and_per_test_id["conn_name"] + + if conn_name == "sqlite_buildin_iris": request.applymarker( pytest.mark.xfail( reason="sqlite_buildin connection does not implement read_sql_table" ) ) - conn = request.getfixturevalue(conn) - iris_frame1 = sql.read_sql_query("SELECT * FROM iris", conn) - iris_frame2 = sql.read_sql("SELECT * FROM iris", conn) + iris_frame1 = sql.read_sql_query(f"SELECT * FROM {iris_uuid}", conn) + iris_frame2 = sql.read_sql(f"SELECT * FROM {iris_uuid}", conn) tm.assert_frame_equal(iris_frame1, iris_frame2) - iris_frame1 = sql.read_sql_table("iris", conn) - iris_frame2 = sql.read_sql("iris", conn) + iris_frame1 = sql.read_sql_table(iris_uuid, conn) + iris_frame2 = sql.read_sql(iris_uuid, conn) tm.assert_frame_equal(iris_frame1, iris_frame2) -def test_not_reflect_all_tables(sqlite_conn): - conn = sqlite_conn +@pytest.mark.parametrize( + "connect_and_uuid", + setup(["sqlite_conn"], table_uuid=["invalid", "other_table"]), + indirect=True, +) +def test_not_reflect_all_tables(connect_and_uuid): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] from sqlalchemy import text from sqlalchemy.engine import Engine + invalid_uuid = table_uuid[0] + other_uuid = table_uuid[1] + # create invalid table query_list = [ - text("CREATE TABLE invalid (x INTEGER, y UNKNOWN);"), - text("CREATE TABLE other_table (x INTEGER, y INTEGER);"), + text(f"CREATE TABLE {invalid_uuid} (x INTEGER, y UNKNOWN);"), + text(f"CREATE TABLE {other_uuid} (x INTEGER, y INTEGER);"), ] for query in query_list: @@ -2416,50 +3054,72 @@ def test_not_reflect_all_tables(sqlite_conn): conn.execute(query) with tm.assert_produces_warning(None): - sql.read_sql_table("other_table", conn) - sql.read_sql_query("SELECT * FROM other_table", conn) + sql.read_sql_table(other_uuid, conn) + sql.read_sql_query(f"SELECT * FROM {other_uuid}", conn) -@pytest.mark.parametrize("conn", all_connectable) -def test_warning_case_insensitive_table_name(conn, request, test_frame1): - conn_name = conn +@pytest.mark.parametrize( + "connect_and_uuid", + setup( + all_connectable, + table_uuid=["test_warning_case_insensitive_table_name", "CaseSensitive"], + ), + indirect=True, +) +def test_warning_case_insensitive_table_name(connect_and_uuid, request, test_frame1): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + table = table_uuid[0] + case_sensitive_uuid = table_uuid[1] + conn_name = connect_and_uuid["conn_name"] + if conn_name == "sqlite_buildin" or "adbc" in conn_name: request.applymarker(pytest.mark.xfail(reason="Does not raise warning")) - conn = request.getfixturevalue(conn) + table_uuid_upper = table.upper() # see gh-7815 with tm.assert_produces_warning( UserWarning, match=( - r"The provided table name 'TABLE1' is not found exactly as such in " + r"The provided table name '{}' is not found exactly as such in " r"the database after writing the table, possibly due to case " r"sensitivity issues. Consider using lower case table names." - ), + ).format(table_uuid_upper), ): with sql.SQLDatabase(conn) as db: - db.check_case_sensitive("TABLE1", "") + db.check_case_sensitive(table_uuid_upper, "") # Test that the warning is certainly NOT triggered in a normal case. with tm.assert_produces_warning(None): - test_frame1.to_sql(name="CaseSensitive", con=conn) + test_frame1.to_sql(name=case_sensitive_uuid, con=conn) -@pytest.mark.parametrize("conn", sqlalchemy_connectable) -def test_sqlalchemy_type_mapping(conn, request): - conn = request.getfixturevalue(conn) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(sqlalchemy_connectable, table_uuid="test_sqlalchemy_type_mapping"), + indirect=True, +) +def test_sqlalchemy_type_mapping(connect_and_uuid, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] from sqlalchemy import TIMESTAMP # Test Timestamp objects (no datetime64 because of timezone) (GH9085) df = DataFrame( {"time": to_datetime(["2014-12-12 01:54", "2014-12-11 02:54"], utc=True)} ) + with sql.SQLDatabase(conn) as db: - table = sql.SQLTable("test_type", db, frame=df) + table = sql.SQLTable(table_uuid, db, frame=df) # GH 9086: TIMESTAMP is the suggested type for datetimes with timezones assert isinstance(table.table.c["time"].type, TIMESTAMP) -@pytest.mark.parametrize("conn", sqlalchemy_connectable) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(sqlalchemy_connectable, table_uuid="test_sqlalchemy_integer_mapping"), + indirect=True, +) @pytest.mark.parametrize( "integer, expected", [ @@ -2480,43 +3140,59 @@ def test_sqlalchemy_type_mapping(conn, request): (int, "BIGINT" if np.dtype(int).name == "int64" else "INTEGER"), ], ) -def test_sqlalchemy_integer_mapping(conn, request, integer, expected): +def test_sqlalchemy_integer_mapping(connect_and_uuid, request, integer, expected): # GH35076 Map pandas integer to optimal SQLAlchemy integer type - conn = request.getfixturevalue(conn) + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + df = DataFrame([0, 1], columns=["a"], dtype=integer) with sql.SQLDatabase(conn) as db: - table = sql.SQLTable("test_type", db, frame=df) + table = sql.SQLTable(table_uuid, db, frame=df) result = str(table.table.c.a.type) assert result == expected -@pytest.mark.parametrize("conn", sqlalchemy_connectable) +@pytest.mark.parametrize( + "connect_and_uuid", + setup( + sqlalchemy_connectable, table_uuid="test_sqlalchemy_integer_overload_mapping" + ), + indirect=True, +) @pytest.mark.parametrize("integer", ["uint64", "UInt64"]) -def test_sqlalchemy_integer_overload_mapping(conn, request, integer): - conn = request.getfixturevalue(conn) +def test_sqlalchemy_integer_overload_mapping(connect_and_uuid, request, integer): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] # GH35076 Map pandas integer to optimal SQLAlchemy integer type df = DataFrame([0, 1], columns=["a"], dtype=integer) with sql.SQLDatabase(conn) as db: with pytest.raises( ValueError, match="Unsigned 64 bit integer datatype is not supported" ): - sql.SQLTable("test_type", db, frame=df) + sql.SQLTable(table_uuid, db, frame=df) -def test_database_uri_string(request, test_frame1): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_database_uri_string"), + indirect=True, +) +def test_database_uri_string(connect_and_uuid, request, test_frame1): pytest.importorskip("sqlalchemy") + table_uuid = connect_and_uuid["table_uuid"] # Test read_sql and .to_sql method with a database URI (GH10654) # db_uri = 'sqlite:///:memory:' # raises # sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) near # "iris": syntax error [SQL: 'iris'] with tm.ensure_clean() as name: db_uri = "sqlite:///" + name - table = "iris" - test_frame1.to_sql(name=table, con=db_uri, if_exists="replace", index=False) - test_frame2 = sql.read_sql(table, db_uri) - test_frame3 = sql.read_sql_table(table, db_uri) - query = "SELECT * FROM iris" + test_frame1.to_sql( + name=table_uuid, con=db_uri, if_exists="replace", index=False + ) + test_frame2 = sql.read_sql(table_uuid, db_uri) + test_frame3 = sql.read_sql_table(table_uuid, db_uri) + query = f"SELECT * FROM {table_uuid}" test_frame4 = sql.read_sql_query(query, db_uri) tm.assert_frame_equal(test_frame1, test_frame2) tm.assert_frame_equal(test_frame1, test_frame3) @@ -2524,59 +3200,78 @@ def test_database_uri_string(request, test_frame1): @td.skip_if_installed("pg8000") -def test_pg8000_sqlalchemy_passthrough_error(request): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_passthrough_error"), + indirect=True, +) +def test_pg8000_sqlalchemy_passthrough_error(connect_and_uuid, request): pytest.importorskip("sqlalchemy") + table_uuid = create_uuid()("sql_alchemy_passthrough_error") # using driver that will not be installed on CI to trigger error # in sqlalchemy.create_engine -> test passing of this error to user db_uri = "postgresql+pg8000://user:pass@host/dbname" with pytest.raises(ImportError, match="pg8000"): - sql.read_sql("select * from table", db_uri) + sql.read_sql(f"select * from {table_uuid}", db_uri) -@pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) -def test_query_by_text_obj(conn, request): +@pytest.mark.parametrize( + "iris_connect_and_per_test_id", setup(sqlalchemy_connectable_iris), indirect=True +) +def test_query_by_text_obj(iris_connect_and_per_test_id, request): # WIP : GH10846 - conn_name = conn - conn = request.getfixturevalue(conn) + conn = iris_connect_and_per_test_id["conn"] + iris_uuid = iris_connect_and_per_test_id["iris_table_uuid"] + conn_name = iris_connect_and_per_test_id["conn_name"] + from sqlalchemy import text if "postgres" in conn_name: - name_text = text('select * from iris where "Name"=:name') + name_text = text(f'select * from {iris_uuid} where "Name"=:name') else: - name_text = text("select * from iris where name=:name") + name_text = text(f"select * from {iris_uuid} where name=:name") iris_df = sql.read_sql(name_text, conn, params={"name": "Iris-versicolor"}) all_names = set(iris_df["Name"]) assert all_names == {"Iris-versicolor"} -@pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) -def test_query_by_select_obj(conn, request): - conn = request.getfixturevalue(conn) +@pytest.mark.parametrize( + "iris_connect_and_per_test_id", setup(sqlalchemy_connectable_iris), indirect=True +) +def test_query_by_select_obj(iris_connect_and_per_test_id, request): + conn = iris_connect_and_per_test_id["conn"] + iris_uuid = iris_connect_and_per_test_id["iris_table_uuid"] # WIP : GH10846 from sqlalchemy import ( bindparam, select, ) - iris = iris_table_metadata() + iris = iris_table_metadata(iris_uuid) name_select = select(iris).where(iris.c.Name == bindparam("name")) iris_df = sql.read_sql(name_select, conn, params={"name": "Iris-setosa"}) all_names = set(iris_df["Name"]) assert all_names == {"Iris-setosa"} -@pytest.mark.parametrize("conn", all_connectable) -def test_column_with_percentage(conn, request): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_column_with_percentage"), + indirect=True, +) +def test_column_with_percentage(connect_and_uuid, request): # GH 37157 - conn_name = conn + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + if conn_name == "sqlite_buildin": request.applymarker(pytest.mark.xfail(reason="Not Implemented")) - conn = request.getfixturevalue(conn) df = DataFrame({"A": [0, 1, 2], "%_variation": [3, 4, 5]}) - df.to_sql(name="test_column_percentage", con=conn, index=False) + df.to_sql(name=table_uuid, con=conn, index=False) - res = sql.read_sql_table("test_column_percentage", conn) + res = sql.read_sql_table(table_uuid, conn) tm.assert_frame_equal(res, df) @@ -2584,10 +3279,11 @@ def test_column_with_percentage(conn, request): def test_sql_open_close(test_frame3): # Test if the IO in the database still work if the connection closed # between the writing and reading (as in many real situations). - with tm.ensure_clean() as name: with contextlib.closing(sqlite3.connect(name)) as conn: - assert sql.to_sql(test_frame3, "test_frame3_legacy", conn, index=False) == 4 + assert ( + sql.to_sql(test_frame3, "test_frame3_legacy", conn, index=False) == 4 + ) with contextlib.closing(sqlite3.connect(name)) as conn: result = sql.read_sql_query("SELECT * FROM test_frame3_legacy;", conn) @@ -2616,19 +3312,28 @@ def close(self): self.conn.close() with contextlib.closing(MockSqliteConnection(":memory:")) as conn: - with tm.assert_produces_warning(UserWarning, match="only supports SQLAlchemy"): + with tm.assert_produces_warning( + UserWarning, match="only supports SQLAlchemy" + ): sql.read_sql("SELECT 1", conn) -def test_sqlite_read_sql_delegate(sqlite_buildin_iris): - conn = sqlite_buildin_iris - iris_frame1 = sql.read_sql_query("SELECT * FROM iris", conn) - iris_frame2 = sql.read_sql("SELECT * FROM iris", conn) +@pytest.mark.parametrize( + "iris_connect_and_per_test_id", setup(["sqlite_buildin_iris"]), indirect=True +) +def test_sqlite_read_sql_delegate(iris_connect_and_per_test_id): + conn = iris_connect_and_per_test_id["conn"] + iris_uuid = iris_connect_and_per_test_id["iris_table_uuid"] + + iris_frame1 = sql.read_sql_query(f"SELECT * FROM {iris_uuid}", conn) + iris_frame2 = sql.read_sql(f"SELECT * FROM {iris_uuid}", conn) tm.assert_frame_equal(iris_frame1, iris_frame2) - msg = "Execution failed on sql 'iris': near \"iris\": syntax error" + msg = ( + f"Execution failed on sql '{iris_uuid}': near \"{iris_uuid}\": syntax error" + ) with pytest.raises(sql.DatabaseError, match=msg): - sql.read_sql("iris", conn) + sql.read_sql(iris_uuid, conn) def test_get_schema2(test_frame1): @@ -2637,14 +3342,20 @@ def test_get_schema2(test_frame1): assert "CREATE" in create_sql -def test_sqlite_type_mapping(sqlite_buildin): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(["sqlite_buildin"], table_uuid="test_sqlite_type_mapping"), + indirect=True, +) +def test_sqlite_type_mapping(connect_and_uuid): # Test Timestamp objects (no datetime64 because of timezone) (GH9085) - conn = sqlite_buildin + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] df = DataFrame( {"time": to_datetime(["2014-12-12 01:54", "2014-12-11 02:54"], utc=True)} ) db = sql.SQLiteDatabase(conn) - table = sql.SQLiteTable("test_type", db, frame=df) + table = sql.SQLiteTable(table_uuid, db, frame=df) schema = table.sql_schema() for col in schema.split("\n"): if col.split()[0].strip('"') == "time": @@ -2655,75 +3366,102 @@ def test_sqlite_type_mapping(sqlite_buildin): # -- Database flavor specific tests -@pytest.mark.parametrize("conn", sqlalchemy_connectable) -def test_create_table(conn, request): - if conn == "sqlite_str": - pytest.skip("sqlite_str has no inspection system") +@pytest.mark.parametrize( + "connect_and_uuid", + setup(sqlalchemy_connectable, table_uuid="test_create_table"), + indirect=True, +) +def test_create_table(connect_and_uuid, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] - conn = request.getfixturevalue(conn) + if conn_name == "sqlite_str": + pytest.skip("sqlite_str has no inspection system") from sqlalchemy import inspect - temp_frame = DataFrame({"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]}) + temp_frame = DataFrame( + {"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]} + ) + with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL: - assert pandasSQL.to_sql(temp_frame, "temp_frame") == 4 + assert pandasSQL.to_sql(temp_frame, table_uuid) == 4 insp = inspect(conn) - assert insp.has_table("temp_frame") + assert insp.has_table(table_uuid) # Cleanup with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL: - pandasSQL.drop_table("temp_frame") + pandasSQL.drop_table(table_uuid) -@pytest.mark.parametrize("conn", sqlalchemy_connectable) -def test_drop_table(conn, request): - if conn == "sqlite_str": - pytest.skip("sqlite_str has no inspection system") +@pytest.mark.parametrize( + "connect_and_uuid", + setup(sqlalchemy_connectable, table_uuid="test_drop_table"), + indirect=True, +) +def test_drop_table(connect_and_uuid, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] - conn = request.getfixturevalue(conn) + if conn_name == "sqlite_str": + pytest.skip("sqlite_str has no inspection system") from sqlalchemy import inspect - temp_frame = DataFrame({"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]}) + temp_frame = DataFrame( + {"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]} + ) with sql.SQLDatabase(conn) as pandasSQL: with pandasSQL.run_transaction(): - assert pandasSQL.to_sql(temp_frame, "temp_frame") == 4 + assert pandasSQL.to_sql(temp_frame, table_uuid) == 4 insp = inspect(conn) - assert insp.has_table("temp_frame") + assert insp.has_table(table_uuid) with pandasSQL.run_transaction(): - pandasSQL.drop_table("temp_frame") + pandasSQL.drop_table(table_uuid) try: insp.clear_cache() # needed with SQLAlchemy 2.0, unavailable prior except AttributeError: pass - assert not insp.has_table("temp_frame") + assert not insp.has_table(table_uuid) -@pytest.mark.parametrize("conn_name", all_connectable) -def test_delete_rows_success(conn_name, test_frame1, request): - table_name = "temp_delete_rows_frame" - conn = request.getfixturevalue(conn_name) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_drop_table"), + indirect=True, +) +def test_delete_rows_success(connect_and_uuid, test_frame1, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] with pandasSQL_builder(conn) as pandasSQL: with pandasSQL.run_transaction(): - assert pandasSQL.to_sql(test_frame1, table_name) == test_frame1.shape[0] + assert pandasSQL.to_sql(test_frame1, table_uuid) == test_frame1.shape[0] with pandasSQL.run_transaction(): - assert pandasSQL.delete_rows(table_name) is None + assert pandasSQL.delete_rows(table_uuid) is None - assert count_rows(conn, table_name) == 0 - assert pandasSQL.has_table(table_name) + assert count_rows(conn, table_uuid) == 0 + assert pandasSQL.has_table(table_uuid) -@pytest.mark.parametrize("conn_name", all_connectable) -def test_delete_rows_is_atomic(conn_name, request): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_delete_rows_atomic"), + indirect=True, +) +def test_delete_rows_is_atomic(connect_and_uuid, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] sqlalchemy = pytest.importorskip("sqlalchemy") - table_name = "temp_delete_rows_atomic_frame" - table_stmt = f"CREATE TABLE {table_name} (a INTEGER, b INTEGER UNIQUE NOT NULL)" + table_stmt = f"CREATE TABLE {table_uuid} (a INTEGER, b INTEGER UNIQUE NOT NULL)" if conn_name != "sqlite_buildin" and "adbc" not in conn_name: table_stmt = sqlalchemy.text(table_stmt) @@ -2732,40 +3470,47 @@ def test_delete_rows_is_atomic(conn_name, request): original_df = DataFrame({"a": [1, 2], "b": [3, 4]}, dtype="int32") replacing_df = DataFrame({"a": [5, 6, 7], "b": [8, 8, 8]}, dtype="int32") - conn = request.getfixturevalue(conn_name) with pandasSQL_builder(conn) as pandasSQL: with pandasSQL.run_transaction() as cur: cur.execute(table_stmt) with pandasSQL.run_transaction(): - pandasSQL.to_sql(original_df, table_name, if_exists="append", index=False) + pandasSQL.to_sql( + original_df, table_uuid, if_exists="append", index=False + ) # inserting duplicated values in a UNIQUE constraint column with pytest.raises(pd.errors.DatabaseError): with pandasSQL.run_transaction(): pandasSQL.to_sql( - replacing_df, table_name, if_exists="delete_rows", index=False + replacing_df, table_uuid, if_exists="delete_rows", index=False ) # failed "delete_rows" is rolled back preserving original data with pandasSQL.run_transaction(): result_df = pandasSQL.read_query( - f"SELECT * FROM {table_name}", dtype="int32" + f"SELECT * FROM {table_uuid}", dtype="int32" ) tm.assert_frame_equal(result_df, original_df) -@pytest.mark.parametrize("conn", all_connectable) -def test_roundtrip(conn, request, test_frame1): - if conn == "sqlite_str": +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_roundtrip"), + indirect=True, +) +def test_roundtrip(connect_and_uuid, request, test_frame1): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + + if conn_name == "sqlite_str": pytest.skip("sqlite_str has no inspection system") - conn_name = conn - conn = request.getfixturevalue(conn) with pandasSQL_builder(conn) as pandasSQL: with pandasSQL.run_transaction(): - assert pandasSQL.to_sql(test_frame1, "test_frame_roundtrip") == 4 - result = pandasSQL.read_query("SELECT * FROM test_frame_roundtrip") + assert pandasSQL.to_sql(test_frame1, table_uuid) == 4 + result = pandasSQL.read_query(f"SELECT * FROM {table_uuid}") if "adbc" in conn_name: result = result.rename(columns={"__index_level_0__": "level_0"}) @@ -2777,44 +3522,70 @@ def test_roundtrip(conn, request, test_frame1): tm.assert_frame_equal(result, test_frame1) -@pytest.mark.parametrize("conn", all_connectable_iris) -def test_execute_sql(conn, request): - conn = request.getfixturevalue(conn) +@pytest.mark.parametrize( + "iris_connect_and_per_test_id", setup(all_connectable_iris), indirect=True +) +def test_execute_sql(iris_connect_and_per_test_id, request): + conn = iris_connect_and_per_test_id["conn"] + iris_uuid = iris_connect_and_per_test_id["iris_table_uuid"] + with pandasSQL_builder(conn) as pandasSQL: with pandasSQL.run_transaction(): - iris_results = pandasSQL.execute("SELECT * FROM iris") + iris_results = pandasSQL.execute(f"SELECT * FROM {iris_uuid}") row = iris_results.fetchone() iris_results.close() assert list(row) == [5.1, 3.5, 1.4, 0.2, "Iris-setosa"] -@pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) -def test_sqlalchemy_read_table(conn, request): - conn = request.getfixturevalue(conn) - iris_frame = sql.read_sql_table("iris", con=conn) +@pytest.mark.parametrize( + "iris_connect_and_per_test_id", setup(sqlalchemy_connectable_iris), indirect=True +) +def test_sqlalchemy_read_table(iris_connect_and_per_test_id, request): + conn = iris_connect_and_per_test_id["conn"] + iris_uuid = iris_connect_and_per_test_id["iris_table_uuid"] + + iris_frame = sql.read_sql_table(iris_uuid, con=conn) check_iris_frame(iris_frame) -@pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) -def test_sqlalchemy_read_table_columns(conn, request): - conn = request.getfixturevalue(conn) +@pytest.mark.parametrize( + "iris_connect_and_per_test_id", setup(sqlalchemy_connectable_iris), indirect=True +) +def test_sqlalchemy_read_table_columns(iris_connect_and_per_test_id, request): + conn = iris_connect_and_per_test_id["conn"] + iris_uuid = iris_connect_and_per_test_id["iris_table_uuid"] + iris_frame = sql.read_sql_table( - "iris", con=conn, columns=["SepalLength", "SepalLength"] + iris_uuid, con=conn, columns=["SepalLength", "SepalLength"] ) - tm.assert_index_equal(iris_frame.columns, Index(["SepalLength", "SepalLength__1"])) + tm.assert_index_equal( + iris_frame.columns, Index(["SepalLength", "SepalLength__1"]) + ) + +@pytest.mark.parametrize( + "iris_connect_and_per_test_id", setup(sqlalchemy_connectable_iris), indirect=True +) +def test_read_table_absent_raises(iris_connect_and_per_test_id, request): + conn = iris_connect_and_per_test_id["conn"] -@pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) -def test_read_table_absent_raises(conn, request): - conn = request.getfixturevalue(conn) msg = "Table this_doesnt_exist not found" with pytest.raises(ValueError, match=msg): sql.read_sql_table("this_doesnt_exist", con=conn) -@pytest.mark.parametrize("conn", sqlalchemy_connectable_types) -def test_sqlalchemy_default_type_conversion(conn, request): - conn_name = conn +@pytest.mark.parametrize( + "connect_and_uuid_types", + setup( + sqlalchemy_connectable_types, + table_uuid="test_sqlalchemy_default_type_conversion", + ), + indirect=True, +) +def test_sqlalchemy_default_type_conversion(connect_and_uuid_types, request): + conn = connect_and_uuid_types["conn"] + table_uuid = connect_and_uuid_types["table_uuid"] + conn_name = connect_and_uuid_types["conn_name"] if conn_name == "sqlite_str": pytest.skip("types tables not created in sqlite_str fixture") elif "mysql" in conn_name or "sqlite" in conn_name: @@ -2822,8 +3593,7 @@ def test_sqlalchemy_default_type_conversion(conn, request): pytest.mark.xfail(reason="boolean dtype not inferred properly") ) - conn = request.getfixturevalue(conn) - df = sql.read_sql_table("types", conn) + df = sql.read_sql_table(table_uuid, conn) assert issubclass(df.FloatCol.dtype.type, np.floating) assert issubclass(df.IntCol.dtype.type, np.integer) @@ -2835,20 +3605,31 @@ def test_sqlalchemy_default_type_conversion(conn, request): assert issubclass(df.BoolColWithNull.dtype.type, object) -@pytest.mark.parametrize("conn", sqlalchemy_connectable) -def test_bigint(conn, request): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(sqlalchemy_connectable, table_uuid="test_bigint"), + indirect=True, +) +def test_bigint(connect_and_uuid, request): # int64 should be converted to BigInteger, GH7433 - conn = request.getfixturevalue(conn) + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] df = DataFrame(data={"i64": [2**62]}) - assert df.to_sql(name="test_bigint", con=conn, index=False) == 1 - result = sql.read_sql_table("test_bigint", conn) + assert df.to_sql(name=table_uuid, con=conn, index=False) == 1 + result = sql.read_sql_table(table_uuid, conn) tm.assert_frame_equal(df, result) -@pytest.mark.parametrize("conn", sqlalchemy_connectable_types) -def test_default_date_load(conn, request): - conn_name = conn +@pytest.mark.parametrize( + "connect_and_uuid_types", + setup(sqlalchemy_connectable_types, table_uuid="test_default_date_load"), + indirect=True, +) +def test_default_date_load(connect_and_uuid_types, request): + conn = connect_and_uuid_types["conn"] + table_uuid = connect_and_uuid_types["table_uuid"] + conn_name = connect_and_uuid_types["conn_name"] if conn_name == "sqlite_str": pytest.skip("types tables not created in sqlite_str fixture") elif "sqlite" in conn_name: @@ -2856,62 +3637,85 @@ def test_default_date_load(conn, request): pytest.mark.xfail(reason="sqlite does not read date properly") ) - conn = request.getfixturevalue(conn) - df = sql.read_sql_table("types", conn) + df = sql.read_sql_table(table_uuid, conn) assert issubclass(df.DateCol.dtype.type, np.datetime64) -@pytest.mark.parametrize("conn", postgresql_connectable) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(postgresql_connectable, table_uuid="test_datetime_with_timezone_query"), + indirect=True, +) @pytest.mark.parametrize("parse_dates", [None, ["DateColWithTz"]]) -def test_datetime_with_timezone_query(conn, request, parse_dates): +def test_datetime_with_timezone_query(connect_and_uuid, request, parse_dates): # edge case that converts postgresql datetime with time zone types # to datetime64[ns,psycopg2.tz.FixedOffsetTimezone..], which is ok # but should be more natural, so coerce to datetime64[ns] for now - conn = request.getfixturevalue(conn) - expected = create_and_load_postgres_datetz(conn) + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + expected = create_and_load_postgres_datetz(conn, table_uuid) # GH11216 - df = read_sql_query("select * from datetz", conn, parse_dates=parse_dates) + df = read_sql_query(f"select * from {table_uuid}", conn, parse_dates=parse_dates) col = df.DateColWithTz tm.assert_series_equal(col, expected) -@pytest.mark.parametrize("conn", postgresql_connectable) -def test_datetime_with_timezone_query_chunksize(conn, request): - conn = request.getfixturevalue(conn) - expected = create_and_load_postgres_datetz(conn) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(postgresql_connectable, table_uuid="test_dt_w_tz_query_chunksize"), + indirect=True, +) +def test_datetime_with_timezone_query_chunksize(connect_and_uuid, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + expected = create_and_load_postgres_datetz(conn, table_uuid) df = concat( - list(read_sql_query("select * from datetz", conn, chunksize=1)), + list(read_sql_query(f"select * from {table_uuid}", conn, chunksize=1)), ignore_index=True, ) col = df.DateColWithTz tm.assert_series_equal(col, expected) -@pytest.mark.parametrize("conn", postgresql_connectable) -def test_datetime_with_timezone_table(conn, request): - conn = request.getfixturevalue(conn) - expected = create_and_load_postgres_datetz(conn) - result = sql.read_sql_table("datetz", conn) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(postgresql_connectable, table_uuid="test_dt_w_tz_table"), + indirect=True, +) +def test_datetime_with_timezone_table(connect_and_uuid, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + expected = create_and_load_postgres_datetz(conn, table_uuid) + result = sql.read_sql_table(table_uuid, conn) exp_frame = expected.to_frame() tm.assert_frame_equal(result, exp_frame) -@pytest.mark.parametrize("conn", sqlalchemy_connectable) -def test_datetime_with_timezone_roundtrip(conn, request): - conn_name = conn - conn = request.getfixturevalue(conn) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(sqlalchemy_connectable, table_uuid="test_dt_w_tz_roundtrip"), + indirect=True, +) +def test_datetime_with_timezone_roundtrip(connect_and_uuid, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] # GH 9086 # Write datetimetz data to a db and read it back # For dbs that support timestamps with timezones, should get back UTC # otherwise naive data should be returned expected = DataFrame( - {"A": date_range("2013-01-01 09:00:00", periods=3, tz="US/Pacific", unit="us")} + { + "A": date_range( + "2013-01-01 09:00:00", periods=3, tz="US/Pacific", unit="us" + ) + } ) - assert expected.to_sql(name="test_datetime_tz", con=conn, index=False) == 3 + assert expected.to_sql(name=table_uuid, con=conn, index=False) == 3 if "postgresql" in conn_name: # SQLAlchemy "timezones" (i.e. offsets) are coerced to UTC @@ -2920,10 +3724,10 @@ def test_datetime_with_timezone_roundtrip(conn, request): # Otherwise, timestamps are returned as local, naive expected["A"] = expected["A"].dt.tz_localize(None) - result = sql.read_sql_table("test_datetime_tz", conn) + result = sql.read_sql_table(table_uuid, conn) tm.assert_frame_equal(result, expected) - result = sql.read_sql_query("SELECT * FROM test_datetime_tz", conn) + result = sql.read_sql_query(f"SELECT * FROM {table_uuid}", conn) if "sqlite" in conn_name: # read_sql_query does not return datetime type like read_sql_table assert isinstance(result.loc[0, "A"], str) @@ -2931,75 +3735,103 @@ def test_datetime_with_timezone_roundtrip(conn, request): tm.assert_frame_equal(result, expected) -@pytest.mark.parametrize("conn", sqlalchemy_connectable) -def test_out_of_bounds_datetime(conn, request): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(sqlalchemy_connectable, table_uuid="test_out_of_bounds_datetime"), + indirect=True, +) +def test_out_of_bounds_datetime(connect_and_uuid, request): # GH 26761 - conn = request.getfixturevalue(conn) + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] data = DataFrame({"date": datetime(9999, 1, 1)}, index=[0]) - assert data.to_sql(name="test_datetime_obb", con=conn, index=False) == 1 - result = sql.read_sql_table("test_datetime_obb", conn) + assert data.to_sql(name=table_uuid, con=conn, index=False) == 1 + result = sql.read_sql_table(table_uuid, conn) expected = DataFrame( np.array([datetime(9999, 1, 1)], dtype="M8[us]"), columns=["date"] ) tm.assert_frame_equal(result, expected) -@pytest.mark.parametrize("conn", sqlalchemy_connectable) -def test_naive_datetimeindex_roundtrip(conn, request): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(sqlalchemy_connectable, table_uuid="test_naive_datetimeindex_roundtrip"), + indirect=True, +) +def test_naive_datetimeindex_roundtrip(connect_and_uuid, request): # GH 23510 # Ensure that a naive DatetimeIndex isn't converted to UTC - conn = request.getfixturevalue(conn) - dates = date_range("2018-01-01", periods=5, freq="6h", unit="us")._with_freq(None) + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + dates = date_range("2018-01-01", periods=5, freq="6h", unit="us")._with_freq( + None + ) expected = DataFrame({"nums": range(5)}, index=dates) - assert expected.to_sql(name="foo_table", con=conn, index_label="info_date") == 5 - result = sql.read_sql_table("foo_table", conn, index_col="info_date") + assert expected.to_sql(name=table_uuid, con=conn, index_label="info_date") == 5 + result = sql.read_sql_table(table_uuid, conn, index_col="info_date") # result index with gain a name from a set_index operation; expected tm.assert_frame_equal(result, expected, check_names=False) -@pytest.mark.parametrize("conn", sqlalchemy_connectable_types) -def test_date_parsing(conn, request): +@pytest.mark.parametrize( + "connect_and_uuid_types", + setup(sqlalchemy_connectable_types, table_uuid="test_date_parsing"), + indirect=True, +) +def test_date_parsing(connect_and_uuid_types, request): # No Parsing - conn_name = conn - conn = request.getfixturevalue(conn) - df = sql.read_sql_table("types", conn) + conn = connect_and_uuid_types["conn"] + table_uuid = connect_and_uuid_types["table_uuid"] + conn_name = connect_and_uuid_types["conn_name"] + + df = sql.read_sql_table(table_uuid, conn) expected_type = object if "sqlite" in conn_name else np.datetime64 assert issubclass(df.DateCol.dtype.type, expected_type) - df = sql.read_sql_table("types", conn, parse_dates=["DateCol"]) + df = sql.read_sql_table(table_uuid, conn, parse_dates=["DateCol"]) assert issubclass(df.DateCol.dtype.type, np.datetime64) - df = sql.read_sql_table("types", conn, parse_dates={"DateCol": "%Y-%m-%d %H:%M:%S"}) + df = sql.read_sql_table( + table_uuid, conn, parse_dates={"DateCol": "%Y-%m-%d %H:%M:%S"} + ) assert issubclass(df.DateCol.dtype.type, np.datetime64) df = sql.read_sql_table( - "types", + table_uuid, conn, parse_dates={"DateCol": {"format": "%Y-%m-%d %H:%M:%S"}}, ) assert issubclass(df.DateCol.dtype.type, np.datetime64) - df = sql.read_sql_table("types", conn, parse_dates=["IntDateCol"]) + df = sql.read_sql_table(table_uuid, conn, parse_dates=["IntDateCol"]) assert issubclass(df.IntDateCol.dtype.type, np.datetime64) - df = sql.read_sql_table("types", conn, parse_dates={"IntDateCol": "s"}) + df = sql.read_sql_table(table_uuid, conn, parse_dates={"IntDateCol": "s"}) assert issubclass(df.IntDateCol.dtype.type, np.datetime64) - df = sql.read_sql_table("types", conn, parse_dates={"IntDateCol": {"unit": "s"}}) + df = sql.read_sql_table( + table_uuid, conn, parse_dates={"IntDateCol": {"unit": "s"}} + ) assert issubclass(df.IntDateCol.dtype.type, np.datetime64) -@pytest.mark.parametrize("conn", sqlalchemy_connectable) -def test_datetime(conn, request): - conn_name = conn - conn = request.getfixturevalue(conn) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(sqlalchemy_connectable, table_uuid="test_datetime"), + indirect=True, +) +def test_datetime(connect_and_uuid, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + df = DataFrame( {"A": date_range("2013-01-01 09:00:00", periods=3), "B": np.arange(3.0)} ) - assert df.to_sql(name="test_datetime", con=conn) == 3 + assert df.to_sql(name=table_uuid, con=conn) == 3 # with read_table -> type information from schema used - result = sql.read_sql_table("test_datetime", conn) + result = sql.read_sql_table(table_uuid, conn) result = result.drop("index", axis=1) expected = df[:] @@ -3007,7 +3839,7 @@ def test_datetime(conn, request): tm.assert_frame_equal(result, expected) # with read_sql -> no type information -> sqlite has no native - result = sql.read_sql_query("SELECT * FROM test_datetime", conn) + result = sql.read_sql_query(f"SELECT * FROM {table_uuid}", conn) result = result.drop("index", axis=1) if "sqlite" in conn_name: assert isinstance(result.loc[0, "A"], str) @@ -3015,24 +3847,29 @@ def test_datetime(conn, request): tm.assert_frame_equal(result, expected) -@pytest.mark.parametrize("conn", sqlalchemy_connectable) -def test_datetime_NaT(conn, request): - conn_name = conn - conn = request.getfixturevalue(conn) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(sqlalchemy_connectable, table_uuid="test_datetime_nat"), + indirect=True, +) +def test_datetime_NaT(connect_and_uuid, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] df = DataFrame( {"A": date_range("2013-01-01 09:00:00", periods=3), "B": np.arange(3.0)} ) df.loc[1, "A"] = np.nan - assert df.to_sql(name="test_datetime", con=conn, index=False) == 3 + assert df.to_sql(name=table_uuid, con=conn, index=False) == 3 # with read_table -> type information from schema used - result = sql.read_sql_table("test_datetime", conn) + result = sql.read_sql_table(table_uuid, conn) expected = df[:] expected["A"] = expected["A"].astype("M8[us]") tm.assert_frame_equal(result, expected) # with read_sql -> no type information -> sqlite has no native - result = sql.read_sql_query("SELECT * FROM test_datetime", conn) + result = sql.read_sql_query(f"SELECT * FROM {table_uuid}", conn) if "sqlite" in conn_name: assert isinstance(result.loc[0, "A"], str) result["A"] = to_datetime(result["A"], errors="coerce") @@ -3040,137 +3877,182 @@ def test_datetime_NaT(conn, request): tm.assert_frame_equal(result, expected) -@pytest.mark.parametrize("conn", sqlalchemy_connectable) -def test_datetime_date(conn, request): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(sqlalchemy_connectable, table_uuid="test_datetime_date"), + indirect=True, +) +def test_datetime_date(connect_and_uuid, request): # test support for datetime.date - conn = request.getfixturevalue(conn) + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] df = DataFrame([date(2014, 1, 1), date(2014, 1, 2)], columns=["a"]) - assert df.to_sql(name="test_date", con=conn, index=False) == 2 - res = read_sql_table("test_date", conn) + assert df.to_sql(name=table_uuid, con=conn, index=False) == 2 + res = read_sql_table(table_uuid, conn) result = res["a"] expected = to_datetime(df["a"]) # comes back as datetime64 tm.assert_series_equal(result, expected) -@pytest.mark.parametrize("conn", sqlalchemy_connectable) -def test_datetime_time(conn, request, sqlite_buildin): +@pytest.mark.parametrize( + "connect_and_uuid", + setup( + sqlalchemy_connectable, + table_uuid=[ + "test_datetime_time_table_a", + "test_datetime_time_table_b", + "test_datetime_time_table_c", + ], + ), + indirect=True, +) +def test_datetime_time(connect_and_uuid, request, sqlite_buildin): # test support for datetime.time - conn_name = conn - conn = request.getfixturevalue(conn) + conn = connect_and_uuid["conn"] + all_tables = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + table_uuid = all_tables[0] + table_uuid2 = all_tables[1] + table_uuid3 = all_tables[2] df = DataFrame([time(9, 0, 0), time(9, 1, 30)], columns=["a"]) - assert df.to_sql(name="test_time", con=conn, index=False) == 2 - res = read_sql_table("test_time", conn) + assert df.to_sql(name=table_uuid, con=conn, index=False) == 2 + res = read_sql_table(table_uuid, conn) tm.assert_frame_equal(res, df) # GH8341 # first, use the fallback to have the sqlite adapter put in place sqlite_conn = sqlite_buildin - assert sql.to_sql(df, "test_time2", sqlite_conn, index=False) == 2 - res = sql.read_sql_query("SELECT * FROM test_time2", sqlite_conn) + assert sql.to_sql(df, table_uuid2, sqlite_conn, index=False) == 2 + res = sql.read_sql_query(f"SELECT * FROM {table_uuid2}", sqlite_conn) ref = df.map(lambda _: _.strftime("%H:%M:%S.%f")) tm.assert_frame_equal(ref, res) # check if adapter is in place # then test if sqlalchemy is unaffected by the sqlite adapter - assert sql.to_sql(df, "test_time3", conn, index=False) == 2 + assert sql.to_sql(df, table_uuid3, conn, index=False) == 2 if "sqlite" in conn_name: - res = sql.read_sql_query("SELECT * FROM test_time3", conn) + res = sql.read_sql_query(f"SELECT * FROM {table_uuid3}", conn) ref = df.map(lambda _: _.strftime("%H:%M:%S.%f")) tm.assert_frame_equal(ref, res) - res = sql.read_sql_table("test_time3", conn) + res = sql.read_sql_table(table_uuid3, conn) tm.assert_frame_equal(df, res) -@pytest.mark.parametrize("conn", sqlalchemy_connectable) -def test_mixed_dtype_insert(conn, request): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(sqlalchemy_connectable, table_uuid="test_mixed_dtype_insert"), + indirect=True, +) +def test_mixed_dtype_insert(connect_and_uuid, request): # see GH6509 - conn = request.getfixturevalue(conn) + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] s1 = Series(2**25 + 1, dtype=np.int32) s2 = Series(0.0, dtype=np.float32) df = DataFrame({"s1": s1, "s2": s2}) # write and read again - assert df.to_sql(name="test_read_write", con=conn, index=False) == 1 - df2 = sql.read_sql_table("test_read_write", conn) + assert df.to_sql(name=table_uuid, con=conn, index=False) == 1 + df2 = sql.read_sql_table(table_uuid, conn) tm.assert_frame_equal(df, df2, check_dtype=False, check_exact=True) -@pytest.mark.parametrize("conn", sqlalchemy_connectable) -def test_nan_numeric(conn, request): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(sqlalchemy_connectable, table_uuid="test_nan_numeric"), + indirect=True, +) +def test_nan_numeric(connect_and_uuid, request): # NaNs in numeric float column - conn = request.getfixturevalue(conn) + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] df = DataFrame({"A": [0, 1, 2], "B": [0.2, np.nan, 5.6]}) - assert df.to_sql(name="test_nan", con=conn, index=False) == 3 + assert df.to_sql(name=table_uuid, con=conn, index=False) == 3 # with read_table - result = sql.read_sql_table("test_nan", conn) + result = sql.read_sql_table(table_uuid, conn) tm.assert_frame_equal(result, df) # with read_sql - result = sql.read_sql_query("SELECT * FROM test_nan", conn) + result = sql.read_sql_query(f"SELECT * FROM {table_uuid}", conn) tm.assert_frame_equal(result, df) -@pytest.mark.parametrize("conn", sqlalchemy_connectable) -def test_nan_fullcolumn(conn, request): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(sqlalchemy_connectable, table_uuid="test_nan_fullcolumn"), + indirect=True, +) +def test_nan_fullcolumn(connect_and_uuid, request): # full NaN column (numeric float column) - conn = request.getfixturevalue(conn) + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] df = DataFrame({"A": [0, 1, 2], "B": [np.nan, np.nan, np.nan]}) - assert df.to_sql(name="test_nan", con=conn, index=False) == 3 + assert df.to_sql(name=table_uuid, con=conn, index=False) == 3 # with read_table - result = sql.read_sql_table("test_nan", conn) + result = sql.read_sql_table(table_uuid, conn) tm.assert_frame_equal(result, df) # with read_sql -> not type info from table -> stays None df["B"] = df["B"].astype("object") df["B"] = None - result = sql.read_sql_query("SELECT * FROM test_nan", conn) + result = sql.read_sql_query(f"SELECT * FROM {table_uuid}", conn) tm.assert_frame_equal(result, df) -@pytest.mark.parametrize("conn", sqlalchemy_connectable) -def test_nan_string(conn, request): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(sqlalchemy_connectable, table_uuid="test_nan_string"), + indirect=True, +) +def test_nan_string(connect_and_uuid, request): # NaNs in string column - conn = request.getfixturevalue(conn) + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] df = DataFrame({"A": [0, 1, 2], "B": ["a", "b", np.nan]}) - assert df.to_sql(name="test_nan", con=conn, index=False) == 3 + assert df.to_sql(name=table_uuid, con=conn, index=False) == 3 # NaNs are coming back as None df.loc[2, "B"] = None # with read_table - result = sql.read_sql_table("test_nan", conn) + result = sql.read_sql_table(table_uuid, conn) tm.assert_frame_equal(result, df) # with read_sql - result = sql.read_sql_query("SELECT * FROM test_nan", conn) + result = sql.read_sql_query(f"SELECT * FROM {table_uuid}", conn) tm.assert_frame_equal(result, df) -@pytest.mark.parametrize("conn", all_connectable) -def test_to_sql_save_index(conn, request): - if "adbc" in conn: +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_to_sql_save_index"), + indirect=True, +) +def test_to_sql_save_index(connect_and_uuid, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + + if "adbc" in conn_name: request.node.add_marker( pytest.mark.xfail( reason="ADBC implementation does not create index", strict=True ) ) - conn_name = conn - conn = request.getfixturevalue(conn) df = DataFrame.from_records( [(1, 2.1, "line1"), (2, 1.5, "line2")], columns=["A", "B", "C"], index=["A"] ) - tbl_name = "test_to_sql_saves_index" with pandasSQL_builder(conn) as pandasSQL: with pandasSQL.run_transaction(): - assert pandasSQL.to_sql(df, tbl_name) == 2 + assert pandasSQL.to_sql(df, table_uuid) == 2 if conn_name in {"sqlite_buildin", "sqlite_str"}: ixs = sql.read_sql_query( "SELECT * FROM sqlite_master WHERE type = 'index' " - f"AND tbl_name = '{tbl_name}'", + f"AND tbl_name = '{table_uuid}'", conn, ) ix_cols = [] @@ -3182,18 +4064,23 @@ def test_to_sql_save_index(conn, request): insp = inspect(conn) - ixs = insp.get_indexes(tbl_name) + ixs = insp.get_indexes(table_uuid) ix_cols = [i["column_names"] for i in ixs] assert ix_cols == [["A"]] -@pytest.mark.parametrize("conn", all_connectable) -def test_transactions(conn, request): - conn_name = conn - conn = request.getfixturevalue(conn) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_transactions"), + indirect=True, +) +def test_transactions(connect_and_uuid, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] - stmt = "CREATE TABLE test_trans (A INT, B TEXT)" + stmt = f"CREATE TABLE {table_uuid} (A INT, B TEXT)" if conn_name != "sqlite_buildin" and "adbc" not in conn_name: from sqlalchemy import text @@ -3204,13 +4091,19 @@ def test_transactions(conn, request): trans.execute(stmt) -@pytest.mark.parametrize("conn", all_connectable) -def test_transaction_rollback(conn, request): - conn_name = conn - conn = request.getfixturevalue(conn) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_transaction_rollback"), + indirect=True, +) +def test_transaction_rollback(connect_and_uuid, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + with pandasSQL_builder(conn) as pandasSQL: with pandasSQL.run_transaction() as trans: - stmt = "CREATE TABLE test_trans (A INT, B TEXT)" + stmt = f"CREATE TABLE {table_uuid} (A INT, B TEXT)" if "adbc" in conn_name or isinstance(pandasSQL, SQLiteDatabase): trans.execute(stmt) else: @@ -3223,7 +4116,7 @@ class DummyException(Exception): pass # Make sure when transaction is rolled back, no rows get inserted - ins_sql = "INSERT INTO test_trans (A,B) VALUES (1, 'blah')" + ins_sql = f"INSERT INTO {table_uuid} (A,B) VALUES (1, 'blah')" if isinstance(pandasSQL, SQLDatabase): from sqlalchemy import text @@ -3236,33 +4129,38 @@ class DummyException(Exception): # ignore raised exception pass with pandasSQL.run_transaction(): - res = pandasSQL.read_query("SELECT * FROM test_trans") + res = pandasSQL.read_query(f"SELECT * FROM {table_uuid}") assert len(res) == 0 # Make sure when transaction is committed, rows do get inserted with pandasSQL.run_transaction() as trans: trans.execute(ins_sql) - res2 = pandasSQL.read_query("SELECT * FROM test_trans") + res2 = pandasSQL.read_query(f"SELECT * FROM {table_uuid}") assert len(res2) == 1 -@pytest.mark.parametrize("conn", sqlalchemy_connectable) -def test_get_schema_create_table(conn, request, test_frame3): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(sqlalchemy_connectable, table_uuid="test_get_schema_create_table"), + indirect=True, +) +def test_get_schema_create_table(connect_and_uuid, request, test_frame3): # Use a dataframe without a bool column, since MySQL converts bool to # TINYINT (which read_sql_table returns as an int and causes a dtype # mismatch) - if conn == "sqlite_str": + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + + if conn_name == "sqlite_str": request.applymarker( pytest.mark.xfail(reason="test does not support sqlite_str fixture") ) - conn = request.getfixturevalue(conn) - from sqlalchemy import text from sqlalchemy.engine import Engine - tbl = "test_get_schema_create_table" - create_sql = sql.get_schema(test_frame3, tbl, con=conn) + create_sql = sql.get_schema(test_frame3, table_uuid, con=conn) blank_test_df = test_frame3.iloc[:0] create_sql = text(create_sql) @@ -3272,66 +4170,86 @@ def test_get_schema_create_table(conn, request, test_frame3): newcon.execute(create_sql) else: conn.execute(create_sql) - returned_df = sql.read_sql_table(tbl, conn) + returned_df = sql.read_sql_table(table_uuid, conn) tm.assert_frame_equal(returned_df, blank_test_df, check_index_type=False) -@pytest.mark.parametrize("conn", sqlalchemy_connectable) -def test_dtype(conn, request): - if conn == "sqlite_str": +@pytest.mark.parametrize( + "connect_and_uuid", + setup( + sqlalchemy_connectable, + table_uuid=[ + "test_dtype_table_a", + "test_dtype_table_b", + "test_dtype_table_c", + "test_dtype_table_single", + "test_dtype_table_error", + ], + ), + indirect=True, +) +def test_dtype(connect_and_uuid): + + conn = connect_and_uuid["conn"] + all_tables = connect_and_uuid["table_uuid"] + table_uuid1 = all_tables[0] + table_uuid2 = all_tables[1] + table_uuid3 = all_tables[2] + table_uuid_single = all_tables[3] + error_table = all_tables[4] + conn_name = connect_and_uuid["conn_name"] + if conn_name == "sqlite_str": pytest.skip("sqlite_str has no inspection system") - conn = request.getfixturevalue(conn) - - from sqlalchemy import ( - TEXT, - String, - ) + from sqlalchemy import TEXT, String, Table from sqlalchemy.schema import MetaData cols = ["A", "B"] data = [(0.8, True), (0.9, None)] df = DataFrame(data, columns=cols) - assert df.to_sql(name="dtype_test", con=conn) == 2 - assert df.to_sql(name="dtype_test2", con=conn, dtype={"B": TEXT}) == 2 + + assert df.to_sql(name=table_uuid1, con=conn) == 2 + assert df.to_sql(name=table_uuid2, con=conn, dtype={"B": TEXT}) == 2 meta = MetaData() - meta.reflect(bind=conn) - sqltype = meta.tables["dtype_test2"].columns["B"].type + table_with_strings = Table(table_uuid2, meta, autoload_with=conn) + sqltype = table_with_strings.columns["B"].type assert isinstance(sqltype, TEXT) msg = "The type of B is not a SQLAlchemy type" with pytest.raises(ValueError, match=msg): - df.to_sql(name="error", con=conn, dtype={"B": str}) + df.to_sql(name=error_table, con=conn, dtype={"B": str}) # GH9083 - assert df.to_sql(name="dtype_test3", con=conn, dtype={"B": String(10)}) == 2 - meta.reflect(bind=conn) - sqltype = meta.tables["dtype_test3"].columns["B"].type + assert df.to_sql(name=table_uuid3, con=conn, dtype={"B": String(10)}) == 2 + meta = MetaData() + table_with_sql_strings = Table(table_uuid3, meta, autoload_with=conn) + sqltype = table_with_sql_strings.columns["B"].type assert isinstance(sqltype, String) assert sqltype.length == 10 # single dtype - assert df.to_sql(name="single_dtype_test", con=conn, dtype=TEXT) == 2 - meta.reflect(bind=conn) - sqltypea = meta.tables["single_dtype_test"].columns["A"].type - sqltypeb = meta.tables["single_dtype_test"].columns["B"].type + assert df.to_sql(name=table_uuid_single, con=conn, dtype=TEXT) == 2 + meta = MetaData() + table_with_sql_dtype_text = Table(table_uuid_single, meta, autoload_with=conn) + sqltypea = table_with_sql_dtype_text.columns["A"].type + sqltypeb = table_with_sql_dtype_text.columns["B"].type assert isinstance(sqltypea, TEXT) assert isinstance(sqltypeb, TEXT) -@pytest.mark.parametrize("conn", sqlalchemy_connectable) -def test_notna_dtype(conn, request): - if conn == "sqlite_str": - pytest.skip("sqlite_str has no inspection system") +@pytest.mark.parametrize( + "connect_and_uuid", + setup(sqlalchemy_connectable, table_uuid="test_notna_dtype"), + indirect=True, +) +def test_notna_dtype(connect_and_uuid): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] - conn_name = conn - conn = request.getfixturevalue(conn) + if conn_name == "sqlite_str": + pytest.skip("sqlite_str has no inspection system") - from sqlalchemy import ( - Boolean, - DateTime, - Float, - Integer, - ) + from sqlalchemy import Boolean, DateTime, Float, Integer, Table from sqlalchemy.schema import MetaData cols = { @@ -3342,31 +4260,33 @@ def test_notna_dtype(conn, request): } df = DataFrame(cols) - tbl = "notna_dtype_test" - assert df.to_sql(name=tbl, con=conn) == 2 - _ = sql.read_sql_table(tbl, conn) + assert df.to_sql(name=table_uuid, con=conn) == 2 + _ = sql.read_sql_table(table_uuid, conn) meta = MetaData() - meta.reflect(bind=conn) + table_with_datatypes = Table(table_uuid, meta, autoload_with=conn) + my_type = Integer if "mysql" in conn_name else Boolean - col_dict = meta.tables[tbl].columns + col_dict = table_with_datatypes.columns assert isinstance(col_dict["Bool"].type, my_type) assert isinstance(col_dict["Date"].type, DateTime) assert isinstance(col_dict["Int"].type, Integer) assert isinstance(col_dict["Float"].type, Float) -@pytest.mark.parametrize("conn", sqlalchemy_connectable) -def test_double_precision(conn, request): - if conn == "sqlite_str": - pytest.skip("sqlite_str has no inspection system") +@pytest.mark.parametrize( + "connect_and_uuid", + setup(sqlalchemy_connectable, table_uuid="test_double_precision"), + indirect=True, +) +def test_double_precision(connect_and_uuid): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] - conn = request.getfixturevalue(conn) + if conn_name == "sqlite_str": + pytest.skip("sqlite_str has no inspection system") - from sqlalchemy import ( - BigInteger, - Float, - Integer, - ) + from sqlalchemy import BigInteger, Float, Integer, Table from sqlalchemy.schema import MetaData V = 1.23456789101112131415 @@ -3383,7 +4303,7 @@ def test_double_precision(conn, request): assert ( df.to_sql( - name="test_dtypes", + name=table_uuid, con=conn, index=False, if_exists="replace", @@ -3391,15 +4311,15 @@ def test_double_precision(conn, request): ) == 1 ) - res = sql.read_sql_table("test_dtypes", conn) + res = sql.read_sql_table(table_uuid, conn) # check precision of float64 assert np.round(df["f64"].iloc[0], 14) == np.round(res["f64"].iloc[0], 14) # check sql types meta = MetaData() - meta.reflect(bind=conn) - col_dict = meta.tables["test_dtypes"].columns + table_with_datatypes = Table(table_uuid, meta, autoload_with=conn) + col_dict = table_with_datatypes.columns assert str(col_dict["f32"].type) == str(col_dict["f64_as_f32"].type) assert isinstance(col_dict["f32"].type, Float) assert isinstance(col_dict["f64"].type, Float) @@ -3407,20 +4327,25 @@ def test_double_precision(conn, request): assert isinstance(col_dict["i64"].type, BigInteger) -@pytest.mark.parametrize("conn", sqlalchemy_connectable) -def test_connectable_issue_example(conn, request): - conn = request.getfixturevalue(conn) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(sqlalchemy_connectable, table_uuid="test_connectable_issue_example"), + indirect=True, +) +def test_connectable_issue_example(connect_and_uuid, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] # This tests the example raised in issue # https://github.com/pandas-dev/pandas/issues/10104 from sqlalchemy.engine import Engine def test_select(connection): - query = "SELECT test_foo_data FROM test_foo_data" + query = f"SELECT test_foo_data FROM {table_uuid}" return sql.read_sql_query(query, con=connection) def test_append(connection, data): - data.to_sql(name="test_foo_data", con=connection, if_exists="append") + data.to_sql(name=table_uuid, con=connection, if_exists="append") def test_connectable(conn): # https://github.com/sqlalchemy/sqlalchemy/commit/ @@ -3437,43 +4362,55 @@ def main(connectable): test_connectable(connectable) assert ( - DataFrame({"test_foo_data": [0, 1, 2]}).to_sql(name="test_foo_data", con=conn) + DataFrame({"test_foo_data": [0, 1, 2]}).to_sql(name=table_uuid, con=conn) == 3 ) main(conn) -@pytest.mark.parametrize("conn", sqlalchemy_connectable) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(sqlalchemy_connectable, table_uuid="test_to_sql_with_negative_npinf"), + indirect=True, +) @pytest.mark.parametrize( "input", [{"foo": [np.inf]}, {"foo": [-np.inf]}, {"foo": [-np.inf], "infe0": ["bar"]}], ) -def test_to_sql_with_negative_npinf(conn, request, input): +def test_to_sql_with_negative_npinf(connect_and_uuid, request, input): # GH 34431 df = DataFrame(input) - conn_name = conn - conn = request.getfixturevalue(conn) - + # conn = connect_and_uuid["conn"] + # conn_name = connect_and_uuid["conn_name"] + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] if "mysql" in conn_name: # GH 36465 # The input {"foo": [-np.inf], "infe0": ["bar"]} does not raise any error # for pymysql version >= 0.10 msg = "Execution failed on sql" with pytest.raises(pd.errors.DatabaseError, match=msg): - df.to_sql(name="foobar", con=conn, index=False) + df.to_sql(name=table_uuid, con=conn, index=False) else: - assert df.to_sql(name="foobar", con=conn, index=False) == 1 - res = sql.read_sql_table("foobar", conn) + assert df.to_sql(name=table_uuid, con=conn, index=False) == 1 + res = sql.read_sql_table(table_uuid, conn) tm.assert_equal(df, res) -@pytest.mark.parametrize("conn", sqlalchemy_connectable) -def test_temporary_table(conn, request): - if conn == "sqlite_str": - pytest.skip("test does not work with str connection") +@pytest.mark.parametrize( + "connect_and_uuid", + setup(sqlalchemy_connectable, table_uuid="test_temporary_table"), + indirect=True, +) +def test_temporary_table(connect_and_uuid, request): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] - conn = request.getfixturevalue(conn) + if conn_name == "sqlite_str": + pytest.skip("test does not work with str connection") from sqlalchemy import ( Column, @@ -3491,7 +4428,7 @@ def test_temporary_table(conn, request): Base = declarative_base() class Temporary(Base): - __tablename__ = "temp_test" + __tablename__ = table_uuid __table_args__ = {"prefixes": ["TEMPORARY"]} id = Column(Integer, primary_key=True) spam = Column(Unicode(30), nullable=False) @@ -3506,64 +4443,86 @@ class Temporary(Base): tm.assert_frame_equal(df, expected) -@pytest.mark.parametrize("conn", all_connectable) -def test_invalid_engine(conn, request, test_frame1): - if conn == "sqlite_buildin" or "adbc" in conn: +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_invalid_engine"), + indirect=True, +) +def test_invalid_engine(connect_and_uuid, request, test_frame1): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + + if conn_name == "sqlite_buildin" or "adbc" in conn_name: request.applymarker( pytest.mark.xfail( reason="SQLiteDatabase/ADBCDatabase does not raise for bad engine" ) ) - conn = request.getfixturevalue(conn) msg = "engine must be one of 'auto', 'sqlalchemy'" with pandasSQL_builder(conn) as pandasSQL: with pytest.raises(ValueError, match=msg): - pandasSQL.to_sql(test_frame1, "test_frame1", engine="bad_engine") + pandasSQL.to_sql(test_frame1, table_uuid, engine="bad_engine") -@pytest.mark.parametrize("conn", all_connectable) -def test_to_sql_with_sql_engine(conn, request, test_frame1): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_to_sql_with_sql_engine"), + indirect=True, +) +def test_to_sql_with_sql_engine(connect_and_uuid, request, test_frame1): """`to_sql` with the `engine` param""" # mostly copied from this class's `_to_sql()` method - conn = request.getfixturevalue(conn) + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] with pandasSQL_builder(conn) as pandasSQL: with pandasSQL.run_transaction(): - assert pandasSQL.to_sql(test_frame1, "test_frame1", engine="auto") == 4 - assert pandasSQL.has_table("test_frame1") + assert pandasSQL.to_sql(test_frame1, table_uuid, engine="auto") == 4 + assert pandasSQL.has_table(table_uuid) num_entries = len(test_frame1) - num_rows = count_rows(conn, "test_frame1") + num_rows = count_rows(conn, table_uuid) assert num_rows == num_entries -@pytest.mark.parametrize("conn", sqlalchemy_connectable) -def test_options_sqlalchemy(conn, request, test_frame1): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(sqlalchemy_connectable, table_uuid="test_options_sqlalchemy"), + indirect=True, +) +def test_options_sqlalchemy(connect_and_uuid, request, test_frame1): # use the set option - conn = request.getfixturevalue(conn) + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] with pd.option_context("io.sql.engine", "sqlalchemy"): with pandasSQL_builder(conn) as pandasSQL: with pandasSQL.run_transaction(): - assert pandasSQL.to_sql(test_frame1, "test_frame1") == 4 - assert pandasSQL.has_table("test_frame1") + assert pandasSQL.to_sql(test_frame1, table_uuid) == 4 + assert pandasSQL.has_table(table_uuid) num_entries = len(test_frame1) - num_rows = count_rows(conn, "test_frame1") + num_rows = count_rows(conn, table_uuid) assert num_rows == num_entries -@pytest.mark.parametrize("conn", all_connectable) -def test_options_auto(conn, request, test_frame1): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_options_auto"), + indirect=True, +) +def test_options_auto(connect_and_uuid, request, test_frame1): # use the set option - conn = request.getfixturevalue(conn) + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] with pd.option_context("io.sql.engine", "auto"): with pandasSQL_builder(conn) as pandasSQL: with pandasSQL.run_transaction(): - assert pandasSQL.to_sql(test_frame1, "test_frame1") == 4 - assert pandasSQL.has_table("test_frame1") + assert pandasSQL.to_sql(test_frame1, table_uuid) == 4 + assert pandasSQL.has_table(table_uuid) num_entries = len(test_frame1) - num_rows = count_rows(conn, "test_frame1") + num_rows = count_rows(conn, table_uuid) assert num_rows == num_entries @@ -3580,10 +4539,14 @@ def test_options_get_engine(): assert isinstance(get_engine("sqlalchemy"), SQLAlchemyEngine) -@pytest.mark.parametrize("conn", all_connectable) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_read_sql_dtype_backend"), + indirect=True, +) @pytest.mark.parametrize("func", ["read_sql", "read_sql_query"]) def test_read_sql_dtype_backend( - conn, + connect_and_uuid, request, string_storage, func, @@ -3592,15 +4555,15 @@ def test_read_sql_dtype_backend( dtype_backend_expected, ): # GH#50048 - conn_name = conn - conn = request.getfixturevalue(conn) - table = "test" + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] df = dtype_backend_data - df.to_sql(name=table, con=conn, index=False, if_exists="replace") + df.to_sql(name=table_uuid, con=conn, index=False, if_exists="replace") with pd.option_context("mode.string_storage", string_storage): result = getattr(pd, func)( - f"Select * from {table}", conn, dtype_backend=dtype_backend + f"Select * from {table_uuid}", conn, dtype_backend=dtype_backend ) expected = dtype_backend_expected(string_storage, dtype_backend, conn_name) @@ -3614,7 +4577,7 @@ def test_read_sql_dtype_backend( with pd.option_context("mode.string_storage", string_storage): iterator = getattr(pd, func)( - f"Select * from {table}", + f"Select * from {table_uuid}", con=conn, dtype_backend=dtype_backend, chunksize=3, @@ -3624,10 +4587,14 @@ def test_read_sql_dtype_backend( tm.assert_frame_equal(result, expected) -@pytest.mark.parametrize("conn", all_connectable) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_read_sql_dtype_backend_table"), + indirect=True, +) @pytest.mark.parametrize("func", ["read_sql", "read_sql_table"]) def test_read_sql_dtype_backend_table( - conn, + connect_and_uuid, request, string_storage, func, @@ -3635,7 +4602,11 @@ def test_read_sql_dtype_backend_table( dtype_backend_data, dtype_backend_expected, ): - if "sqlite" in conn and "adbc" not in conn: + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + + if "sqlite" in conn_name and "adbc" not in conn_name: request.applymarker( pytest.mark.xfail( reason=( @@ -3645,14 +4616,11 @@ def test_read_sql_dtype_backend_table( ) ) # GH#50048 - conn_name = conn - conn = request.getfixturevalue(conn) - table = "test" df = dtype_backend_data - df.to_sql(name=table, con=conn, index=False, if_exists="replace") + df.to_sql(name=table_uuid, con=conn, index=False, if_exists="replace") with pd.option_context("mode.string_storage", string_storage): - result = getattr(pd, func)(table, conn, dtype_backend=dtype_backend) + result = getattr(pd, func)(table_uuid, conn, dtype_backend=dtype_backend) expected = dtype_backend_expected(string_storage, dtype_backend, conn_name) tm.assert_frame_equal(result, expected) @@ -3662,7 +4630,7 @@ def test_read_sql_dtype_backend_table( with pd.option_context("mode.string_storage", string_storage): iterator = getattr(pd, func)( - table, + table_uuid, conn, dtype_backend=dtype_backend, chunksize=3, @@ -3672,20 +4640,26 @@ def test_read_sql_dtype_backend_table( tm.assert_frame_equal(result, expected) -@pytest.mark.parametrize("conn", all_connectable) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_read_sql_invalid_dtype_backend_table"), + indirect=True, +) @pytest.mark.parametrize("func", ["read_sql", "read_sql_table", "read_sql_query"]) -def test_read_sql_invalid_dtype_backend_table(conn, request, func, dtype_backend_data): - conn = request.getfixturevalue(conn) - table = "test" +def test_read_sql_invalid_dtype_backend_table( + connect_and_uuid, request, func, dtype_backend_data +): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] df = dtype_backend_data - df.to_sql(name=table, con=conn, index=False, if_exists="replace") + df.to_sql(name=table_uuid, con=conn, index=False, if_exists="replace") msg = ( "dtype_backend numpy is invalid, only 'numpy_nullable' and " "'pyarrow' are allowed." ) with pytest.raises(ValueError, match=msg): - getattr(pd, func)(table, conn, dtype_backend="numpy") + getattr(pd, func)(table_uuid, conn, dtype_backend="numpy") @pytest.fixture @@ -3749,21 +4723,28 @@ def func(string_storage, dtype_backend, conn_name) -> DataFrame: return func -@pytest.mark.parametrize("conn", all_connectable) -def test_chunksize_empty_dtypes(conn, request): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_chunksize_empty_dtypes"), + indirect=True, +) +def test_chunksize_empty_dtypes(connect_and_uuid, request): # GH#50245 - if "adbc" in conn: + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + conn_name = connect_and_uuid["conn_name"] + + if "adbc" in conn_name: request.node.add_marker( pytest.mark.xfail(reason="chunksize argument NotImplemented with ADBC") ) - conn = request.getfixturevalue(conn) dtypes = {"a": "int64", "b": "object"} df = DataFrame(columns=["a", "b"]).astype(dtypes) expected = df.copy() - df.to_sql(name="test", con=conn, index=False, if_exists="replace") + df.to_sql(name=table_uuid, con=conn, index=False, if_exists="replace") for result in read_sql_query( - "SELECT * FROM test", + f"SELECT * FROM {table_uuid}", conn, dtype=dtypes, chunksize=1, @@ -3771,18 +4752,22 @@ def test_chunksize_empty_dtypes(conn, request): tm.assert_frame_equal(result, expected) -@pytest.mark.parametrize("conn", all_connectable) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(all_connectable, table_uuid="test_read_sql_dtype"), + indirect=True, +) @pytest.mark.parametrize("dtype_backend", [lib.no_default, "numpy_nullable"]) @pytest.mark.parametrize("func", ["read_sql", "read_sql_query"]) -def test_read_sql_dtype(conn, request, func, dtype_backend): +def test_read_sql_dtype(connect_and_uuid, request, func, dtype_backend): # GH#50797 - conn = request.getfixturevalue(conn) - table = "test" + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] df = DataFrame({"a": [1, 2, 3], "b": 5}) - df.to_sql(name=table, con=conn, index=False, if_exists="replace") + df.to_sql(name=table_uuid, con=conn, index=False, if_exists="replace") result = getattr(pd, func)( - f"Select * from {table}", + f"Select * from {table_uuid}", conn, dtype={"a": np.float64}, dtype_backend=dtype_backend, @@ -3799,25 +4784,42 @@ def test_read_sql_dtype(conn, request, func, dtype_backend): tm.assert_frame_equal(result, expected) -def test_bigint_warning(sqlite_engine): - conn = sqlite_engine +@pytest.mark.parametrize( + "connect_and_uuid", + setup(["sqlite_engine"], table_uuid="test_bigint_warning"), + indirect=True, +) +def test_bigint_warning(connect_and_uuid): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] # test no warning for BIGINT (to support int64) is raised (GH7433) df = DataFrame({"a": [1, 2]}, dtype="int64") - assert df.to_sql(name="test_bigintwarning", con=conn, index=False) == 2 + assert df.to_sql(name=table_uuid, con=conn, index=False) == 2 with tm.assert_produces_warning(None): - sql.read_sql_table("test_bigintwarning", conn) + sql.read_sql_table(table_uuid, conn) -def test_valueerror_exception(sqlite_engine): - conn = sqlite_engine +@pytest.mark.parametrize( + "connect_and_uuid", + setup(["sqlite_engine"], table_uuid="test_valueerror_exception"), + indirect=True, +) +def test_valueerror_exception(connect_and_uuid): + conn = connect_and_uuid["conn"] df = DataFrame({"col1": [1, 2], "col2": [3, 4]}) with pytest.raises(ValueError, match="Empty table name specified"): df.to_sql(name="", con=conn, if_exists="replace", index=False) -def test_row_object_is_named_tuple(sqlite_engine): - conn = sqlite_engine +@pytest.mark.parametrize( + "connect_and_uuid", + setup(["sqlite_engine"], table_uuid="test_row_object_is_named_tuple"), + indirect=True, +) +def test_row_object_is_named_tuple(connect_and_uuid): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] # GH 40682 # Test for the is_named_tuple() function # Placed here due to its usage of sqlalchemy @@ -3835,7 +4837,7 @@ def test_row_object_is_named_tuple(sqlite_engine): BaseModel = declarative_base() class Test(BaseModel): - __tablename__ = "test_frame" + __tablename__ = table_uuid id = Column(Integer, primary_key=True) string_column = Column(String(50)) @@ -3845,7 +4847,7 @@ class Test(BaseModel): with Session() as session: df = DataFrame({"id": [0, 1], "string_column": ["hello", "world"]}) assert ( - df.to_sql(name="test_frame", con=conn, index=False, if_exists="replace") + df.to_sql(name=table_uuid, con=conn, index=False, if_exists="replace") == 2 ) session.commit() @@ -3855,15 +4857,20 @@ class Test(BaseModel): assert list(df.columns) == ["id", "string_column"] -def test_read_sql_string_inference(sqlite_engine): - conn = sqlite_engine +@pytest.mark.parametrize( + "connect_and_uuid", + setup(["sqlite_engine"], table_uuid="test_read_sql_string_inference"), + indirect=True, +) +def test_read_sql_string_inference(connect_and_uuid): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] # GH#54430 - table = "test" df = DataFrame({"a": ["x", "y"]}) - df.to_sql(table, con=conn, index=False, if_exists="replace") + df.to_sql(table_uuid, con=conn, index=False, if_exists="replace") with pd.option_context("future.infer_string", True): - result = read_sql_table(table, conn) + result = read_sql_table(table_uuid, conn) dtype = pd.StringDtype(na_value=np.nan) expected = DataFrame( @@ -3873,12 +4880,18 @@ def test_read_sql_string_inference(sqlite_engine): tm.assert_frame_equal(result, expected) -def test_roundtripping_datetimes(sqlite_engine): - conn = sqlite_engine +@pytest.mark.parametrize( + "connect_and_uuid", + setup(["sqlite_engine"], table_uuid="test_roundtripping_datetimes"), + indirect=True, +) +def test_roundtripping_datetimes(connect_and_uuid): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] # GH#54877 df = DataFrame({"t": [datetime(2020, 12, 31, 12)]}, dtype="datetime64[ns]") - df.to_sql("test", conn, if_exists="replace", index=False) - result = pd.read_sql("select * from test", conn).iloc[0, 0] + df.to_sql(table_uuid, conn, if_exists="replace", index=False) + result = pd.read_sql(f"select * from {table_uuid}", conn).iloc[0, 0] assert result == "2020-12-31 12:00:00.000000" @@ -3891,19 +4904,39 @@ def sqlite_builtin_detect_types(): yield conn -def test_roundtripping_datetimes_detect_types(sqlite_builtin_detect_types): +@pytest.mark.parametrize( + "connect_and_uuid", + setup( + ["sqlite_builtin_detect_types"], table_uuid="test_rt_datetimes_detect_types" + ), + indirect=True, +) +def test_roundtripping_datetimes_detect_types(connect_and_uuid): # https://github.com/pandas-dev/pandas/issues/55554 - conn = sqlite_builtin_detect_types + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] df = DataFrame({"t": [datetime(2020, 12, 31, 12)]}, dtype="datetime64[ns]") - df.to_sql("test", conn, if_exists="replace", index=False) - result = pd.read_sql("select * from test", conn).iloc[0, 0] + df.to_sql(table_uuid, conn, if_exists="replace", index=False) + result = pd.read_sql(f"select * from {table_uuid}", conn).iloc[0, 0] assert result == Timestamp("2020-12-31 12:00:00.000000") +@pytest.mark.parametrize( + "connect_and_uuid", + setup( + ["postgresql_psycopg2_engine"], + table_uuid=[ + "schema_public_uuid", + "schema_public_explicit_uuid", + "schema_other_uuid", + ], + ), + indirect=True, +) @pytest.mark.db -def test_psycopg2_schema_support(postgresql_psycopg2_engine): - conn = postgresql_psycopg2_engine - +def test_psycopg2_schema_support(connect_and_uuid): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] # only test this for postgresql (schema's not supported in # mysql/sqlite) df = DataFrame({"col1": [1, 2], "col2": [0.1, 0.2], "col3": ["a", "n"]}) @@ -3914,11 +4947,15 @@ def test_psycopg2_schema_support(postgresql_psycopg2_engine): con.exec_driver_sql("DROP SCHEMA IF EXISTS other CASCADE;") con.exec_driver_sql("CREATE SCHEMA other;") + schema_public_uuid = table_uuid[0] + schema_public_explicit_uuid = table_uuid[1] + schema_other_uuid = table_uuid[2] + # write dataframe to different schema's - assert df.to_sql(name="test_schema_public", con=conn, index=False) == 2 + assert df.to_sql(name=schema_public_uuid, con=conn, index=False) == 2 assert ( df.to_sql( - name="test_schema_public_explicit", + name=schema_public_explicit_uuid, con=conn, index=False, schema="public", @@ -3926,21 +4963,21 @@ def test_psycopg2_schema_support(postgresql_psycopg2_engine): == 2 ) assert ( - df.to_sql(name="test_schema_other", con=conn, index=False, schema="other") == 2 + df.to_sql(name=schema_other_uuid, con=conn, index=False, schema="other") == 2 ) # read dataframes back in - res1 = sql.read_sql_table("test_schema_public", conn) + res1 = sql.read_sql_table(schema_public_uuid, conn) tm.assert_frame_equal(df, res1) - res2 = sql.read_sql_table("test_schema_public_explicit", conn) + res2 = sql.read_sql_table(schema_public_explicit_uuid, conn) tm.assert_frame_equal(df, res2) - res3 = sql.read_sql_table("test_schema_public_explicit", conn, schema="public") + res3 = sql.read_sql_table(schema_public_explicit_uuid, conn, schema="public") tm.assert_frame_equal(df, res3) - res4 = sql.read_sql_table("test_schema_other", conn, schema="other") + res4 = sql.read_sql_table(schema_other_uuid, conn, schema="other") tm.assert_frame_equal(df, res4) - msg = "Table test_schema_other not found" + msg = f"Table {schema_other_uuid} not found" with pytest.raises(ValueError, match=msg): - sql.read_sql_table("test_schema_other", conn, schema="public") + sql.read_sql_table(schema_other_uuid, conn, schema="public") # different if_exists options @@ -3952,10 +4989,10 @@ def test_psycopg2_schema_support(postgresql_psycopg2_engine): # write dataframe with different if_exists options assert ( - df.to_sql(name="test_schema_other", con=conn, schema="other", index=False) == 2 + df.to_sql(name=schema_other_uuid, con=conn, schema="other", index=False) == 2 ) df.to_sql( - name="test_schema_other", + name=schema_other_uuid, con=conn, schema="other", index=False, @@ -3963,7 +5000,7 @@ def test_psycopg2_schema_support(postgresql_psycopg2_engine): ) assert ( df.to_sql( - name="test_schema_other", + name=schema_other_uuid, con=conn, schema="other", index=False, @@ -3971,25 +5008,31 @@ def test_psycopg2_schema_support(postgresql_psycopg2_engine): ) == 2 ) - res = sql.read_sql_table("test_schema_other", conn, schema="other") + res = sql.read_sql_table(schema_other_uuid, conn, schema="other") tm.assert_frame_equal(concat([df, df], ignore_index=True), res) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(["postgresql_psycopg2_engine"], table_uuid="test_self_join_date_columns"), + indirect=True, +) @pytest.mark.db -def test_self_join_date_columns(postgresql_psycopg2_engine): +def test_self_join_date_columns(connect_and_uuid): # GH 44421 - conn = postgresql_psycopg2_engine + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] from sqlalchemy.sql import text create_table = text( - """ - CREATE TABLE person + f""" + CREATE TABLE {table_uuid} ( - id serial constraint person_pkey primary key, + id serial constraint {table_uuid}_pkey primary key, created_dt timestamp with time zone ); - INSERT INTO person + INSERT INTO {table_uuid} VALUES (1, '2021-01-01T00:00:00Z'); """ ) @@ -3997,9 +5040,10 @@ def test_self_join_date_columns(postgresql_psycopg2_engine): with con.begin(): con.execute(create_table) - sql_query = ( - 'SELECT * FROM "person" AS p1 INNER JOIN "person" AS p2 ON p1.id = p2.id;' - ) + sql_query = f''' + SELECT * FROM "{table_uuid}" + AS p1 INNER JOIN "{table_uuid}" + AS p2 ON p1.id = p2.id;''' result = pd.read_sql(sql_query, conn) expected = DataFrame( [[1, Timestamp("2021", tz="UTC")] * 2], columns=["id", "created_dt"] * 2 @@ -4009,36 +5053,56 @@ def test_self_join_date_columns(postgresql_psycopg2_engine): # Cleanup with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL: - pandasSQL.drop_table("person") + pandasSQL.drop_table(table_uuid) -def test_create_and_drop_table(sqlite_engine): - conn = sqlite_engine - temp_frame = DataFrame({"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]}) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(["sqlite_engine"], table_uuid="test_create_and_drop_table"), + indirect=True, +) +def test_create_and_drop_table(connect_and_uuid): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + temp_frame = DataFrame( + {"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]} + ) with sql.SQLDatabase(conn) as pandasSQL: with pandasSQL.run_transaction(): - assert pandasSQL.to_sql(temp_frame, "drop_test_frame") == 4 + assert pandasSQL.to_sql(temp_frame, table_uuid) == 4 - assert pandasSQL.has_table("drop_test_frame") + assert pandasSQL.has_table(table_uuid) with pandasSQL.run_transaction(): - pandasSQL.drop_table("drop_test_frame") + pandasSQL.drop_table(table_uuid) - assert not pandasSQL.has_table("drop_test_frame") + assert not pandasSQL.has_table(table_uuid) -def test_sqlite_datetime_date(sqlite_buildin): - conn = sqlite_buildin +@pytest.mark.parametrize( + "connect_and_uuid", + setup(["sqlite_buildin"], table_uuid="test_sqlite_datetime_date"), + indirect=True, +) +def test_sqlite_datetime_date(connect_and_uuid): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] df = DataFrame([date(2014, 1, 1), date(2014, 1, 2)], columns=["a"]) - assert df.to_sql(name="test_date", con=conn, index=False) == 2 - res = read_sql_query("SELECT * FROM test_date", conn) + assert df.to_sql(name=table_uuid, con=conn, index=False) == 2 + res = read_sql_query(f"SELECT * FROM {table_uuid}", conn) # comes back as strings tm.assert_frame_equal(res, df.astype(str)) +@pytest.mark.parametrize( + "connect_and_uuid", + setup(["sqlite_buildin"], table_uuid="test_sqlite_datetime_time"), + indirect=True, +) @pytest.mark.parametrize("tz_aware", [False, True]) -def test_sqlite_datetime_time(tz_aware, sqlite_buildin): - conn = sqlite_buildin +def test_sqlite_datetime_time(tz_aware, connect_and_uuid): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] # test support for datetime.time, GH #8341 if not tz_aware: tz_times = [time(9, 0, 0), time(9, 1, 30)] @@ -4048,8 +5112,8 @@ def test_sqlite_datetime_time(tz_aware, sqlite_buildin): df = DataFrame(tz_times, columns=["a"]) - assert df.to_sql(name="test_time", con=conn, index=False) == 2 - res = read_sql_query("SELECT * FROM test_time", conn) + assert df.to_sql(name=table_uuid, con=conn, index=False) == 2 + res = read_sql_query(f"SELECT * FROM {table_uuid}", conn) # comes back as strings expected = df.map(lambda _: _.strftime("%H:%M:%S.%f")) tm.assert_frame_equal(res, expected) @@ -4063,30 +5127,54 @@ def get_sqlite_column_type(conn, table, column): raise ValueError(f"Table {table}, column {column} not found") -def test_sqlite_test_dtype(sqlite_buildin): - conn = sqlite_buildin +@pytest.mark.parametrize( + "connect_and_uuid", + setup( + ["sqlite_buildin"], + table_uuid=[ + "test_sqlite_dtype_table_1", + "test_sqlite_dtype_table_1", + "test_sqlite_dtype_table_error", + "test_sqlite_dtype_table_single", + ], + ), + indirect=True, +) +def test_sqlite_test_dtype(connect_and_uuid): + conn = connect_and_uuid["conn"] + tables_list = connect_and_uuid["table_uuid"] + table_uuid1 = tables_list[0] + table_uuid2 = tables_list[1] + table_error = tables_list[2] + table_single = tables_list[3] cols = ["A", "B"] data = [(0.8, True), (0.9, None)] df = DataFrame(data, columns=cols) - assert df.to_sql(name="dtype_test", con=conn) == 2 - assert df.to_sql(name="dtype_test2", con=conn, dtype={"B": "STRING"}) == 2 + assert df.to_sql(name=table_uuid1, con=conn) == 2 + assert df.to_sql(name=table_uuid2, con=conn, dtype={"B": "STRING"}) == 2 # sqlite stores Boolean values as INTEGER - assert get_sqlite_column_type(conn, "dtype_test", "B") == "INTEGER" + assert get_sqlite_column_type(conn, table_uuid1, "B") == "INTEGER" - assert get_sqlite_column_type(conn, "dtype_test2", "B") == "STRING" + assert get_sqlite_column_type(conn, table_uuid2, "B") == "STRING" msg = r"B \(\) not a string" with pytest.raises(ValueError, match=msg): - df.to_sql(name="error", con=conn, dtype={"B": bool}) + df.to_sql(name=table_error, con=conn, dtype={"B": bool}) # single dtype - assert df.to_sql(name="single_dtype_test", con=conn, dtype="STRING") == 2 - assert get_sqlite_column_type(conn, "single_dtype_test", "A") == "STRING" - assert get_sqlite_column_type(conn, "single_dtype_test", "B") == "STRING" + assert df.to_sql(name=table_single, con=conn, dtype="STRING") == 2 + assert get_sqlite_column_type(conn, table_single, "A") == "STRING" + assert get_sqlite_column_type(conn, table_single, "B") == "STRING" -def test_sqlite_notna_dtype(sqlite_buildin): - conn = sqlite_buildin +@pytest.mark.parametrize( + "connect_and_uuid", + setup(["sqlite_buildin"], table_uuid="test_sqlite_notna_dtype"), + indirect=True, +) +def test_sqlite_notna_dtype(connect_and_uuid): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] cols = { "Bool": Series([True, None]), "Date": Series([datetime(2012, 5, 1), None]), @@ -4095,18 +5183,22 @@ def test_sqlite_notna_dtype(sqlite_buildin): } df = DataFrame(cols) - tbl = "notna_dtype_test" - assert df.to_sql(name=tbl, con=conn) == 2 + assert df.to_sql(name=table_uuid, con=conn) == 2 - assert get_sqlite_column_type(conn, tbl, "Bool") == "INTEGER" - assert get_sqlite_column_type(conn, tbl, "Date") == "TIMESTAMP" - assert get_sqlite_column_type(conn, tbl, "Int") == "INTEGER" - assert get_sqlite_column_type(conn, tbl, "Float") == "REAL" + assert get_sqlite_column_type(conn, table_uuid, "Bool") == "INTEGER" + assert get_sqlite_column_type(conn, table_uuid, "Date") == "TIMESTAMP" + assert get_sqlite_column_type(conn, table_uuid, "Int") == "INTEGER" + assert get_sqlite_column_type(conn, table_uuid, "Float") == "REAL" -def test_sqlite_illegal_names(sqlite_buildin): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(["sqlite_buildin"], table_uuid="test_sqlite_illegal_names"), + indirect=True, +) +def test_sqlite_illegal_names(connect_and_uuid): # For sqlite, these should work fine - conn = sqlite_buildin + conn = connect_and_uuid["conn"] df = DataFrame([[1, 2], [3, 4]], columns=["a", "b"]) msg = "Empty table or column name specified" @@ -4166,14 +5258,27 @@ def tquery(query, con=None): return None if res is None else list(res) -def test_xsqlite_basic(sqlite_buildin): +@pytest.mark.parametrize( + "connect_and_uuid", + setup( + ["sqlite_buildin"], + table_uuid=["test_xsqlite_basic_table_a", "test_xsqlite_basic_table_b"], + ), + indirect=True, +) +def test_xsqlite_basic(connect_and_uuid): + conn = connect_and_uuid["conn"] + all_tables = connect_and_uuid["table_uuid"] + table_uuid1 = all_tables[0] + table_uuid2 = all_tables[1] + frame = DataFrame( np.random.default_rng(2).standard_normal((10, 4)), columns=Index(list("ABCD")), index=date_range("2000-01-01", periods=10, freq="B"), ) - assert sql.to_sql(frame, name="test_table", con=sqlite_buildin, index=False) == 10 - result = sql.read_sql("select * from test_table", sqlite_buildin) + assert sql.to_sql(frame, name=table_uuid1, con=conn, index=False) == 10 + result = sql.read_sql(f"select * from {table_uuid1}", conn) # HACK! Change this once indexes are handled properly. result.index = frame.index @@ -4185,81 +5290,111 @@ def test_xsqlite_basic(sqlite_buildin): frame2 = frame.copy() new_idx = Index(np.arange(len(frame2)), dtype=np.int64) + 10 frame2["Idx"] = new_idx.copy() - assert sql.to_sql(frame2, name="test_table2", con=sqlite_buildin, index=False) == 10 - result = sql.read_sql("select * from test_table2", sqlite_buildin, index_col="Idx") + assert sql.to_sql(frame2, name=table_uuid2, con=conn, index=False) == 10 + result = sql.read_sql(f"select * from {table_uuid2}", conn, index_col="Idx") expected = frame.copy() expected.index = new_idx expected.index.name = "Idx" tm.assert_frame_equal(expected, result) -def test_xsqlite_write_row_by_row(sqlite_buildin): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(["sqlite_buildin"], table_uuid="test_xsqlite_write_row_by_row"), + indirect=True, +) +def test_xsqlite_write_row_by_row(connect_and_uuid): frame = DataFrame( np.random.default_rng(2).standard_normal((10, 4)), columns=Index(list("ABCD")), index=date_range("2000-01-01", periods=10, freq="B"), ) + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] frame.iloc[0, 0] = np.nan - create_sql = sql.get_schema(frame, "test") - cur = sqlite_buildin.cursor() + create_sql = sql.get_schema(frame, table_uuid) + cur = conn.cursor() cur.execute(create_sql) - ins = "INSERT INTO test VALUES (%s, %s, %s, %s)" + ins = f"INSERT INTO {table_uuid} VALUES (%s, %s, %s, %s)" for _, row in frame.iterrows(): fmt_sql = format_query(ins, *row) - tquery(fmt_sql, con=sqlite_buildin) + tquery(fmt_sql, con=conn) - sqlite_buildin.commit() + conn.commit() - result = sql.read_sql("select * from test", con=sqlite_buildin) + result = sql.read_sql(f"select * from {table_uuid}", con=conn) result.index = frame.index tm.assert_frame_equal(result, frame, rtol=1e-3) -def test_xsqlite_execute(sqlite_buildin): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(["sqlite_buildin"], table_uuid="test_xsqlite_execute"), + indirect=True, +) +def test_xsqlite_execute(connect_and_uuid): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + frame = DataFrame( np.random.default_rng(2).standard_normal((10, 4)), columns=Index(list("ABCD")), index=date_range("2000-01-01", periods=10, freq="B"), ) - create_sql = sql.get_schema(frame, "test") - cur = sqlite_buildin.cursor() + create_sql = sql.get_schema(frame, table_uuid) + cur = conn.cursor() cur.execute(create_sql) - ins = "INSERT INTO test VALUES (?, ?, ?, ?)" + ins = f"INSERT INTO {table_uuid} VALUES (?, ?, ?, ?)" row = frame.iloc[0] - with sql.pandasSQL_builder(sqlite_buildin) as pandas_sql: + with sql.pandasSQL_builder(conn) as pandas_sql: pandas_sql.execute(ins, tuple(row)) - sqlite_buildin.commit() + conn.commit() - result = sql.read_sql("select * from test", sqlite_buildin) + result = sql.read_sql(f"select * from {table_uuid}", conn) result.index = frame.index[:1] tm.assert_frame_equal(result, frame[:1]) -def test_xsqlite_schema(sqlite_buildin): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(["sqlite_buildin"], table_uuid="test_xsqlite_schema"), + indirect=True, +) +def test_xsqlite_schema(connect_and_uuid): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + frame = DataFrame( np.random.default_rng(2).standard_normal((10, 4)), columns=Index(list("ABCD")), index=date_range("2000-01-01", periods=10, freq="B"), ) - create_sql = sql.get_schema(frame, "test") + create_sql = sql.get_schema(frame, table_uuid) lines = create_sql.splitlines() for line in lines: tokens = line.split(" ") if len(tokens) == 2 and tokens[0] == "A": assert tokens[1] == "DATETIME" - create_sql = sql.get_schema(frame, "test", keys=["A", "B"]) + create_sql = sql.get_schema(frame, table_uuid, keys=["A", "B"]) lines = create_sql.splitlines() assert 'PRIMARY KEY ("A", "B")' in create_sql - cur = sqlite_buildin.cursor() + cur = conn.cursor() cur.execute(create_sql) -def test_xsqlite_execute_fail(sqlite_buildin): - create_sql = """ - CREATE TABLE test +@pytest.mark.parametrize( + "connect_and_uuid", + setup(["sqlite_buildin"], table_uuid="test_xsqlite_execute_fail"), + indirect=True, +) +def test_xsqlite_execute_fail(connect_and_uuid): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + create_sql = f""" + CREATE TABLE {table_uuid} ( a TEXT, b TEXT, @@ -4267,20 +5402,23 @@ def test_xsqlite_execute_fail(sqlite_buildin): PRIMARY KEY (a, b) ); """ - cur = sqlite_buildin.cursor() + cur = conn.cursor() cur.execute(create_sql) - with sql.pandasSQL_builder(sqlite_buildin) as pandas_sql: - pandas_sql.execute("INSERT INTO test VALUES('foo', 'bar', 1.234)") - pandas_sql.execute("INSERT INTO test VALUES('foo', 'baz', 2.567)") + with sql.pandasSQL_builder(conn) as pandas_sql: + pandas_sql.execute(f"INSERT INTO {table_uuid} VALUES('foo', 'bar', 1.234)") + pandas_sql.execute(f"INSERT INTO {table_uuid} VALUES('foo', 'baz', 2.567)") with pytest.raises(sql.DatabaseError, match="Execution failed on sql"): - pandas_sql.execute("INSERT INTO test VALUES('foo', 'bar', 7)") + pandas_sql.execute(f"INSERT INTO {table_uuid} VALUES('foo', 'bar', 7)") def test_xsqlite_execute_closed_connection(): - create_sql = """ - CREATE TABLE test + # This test should not be used with the connect_and_uuid fixture, because that + # fixture automatically manages connection closing + table_uuid = "table_" + create_uuid()("test") + create_sql = f""" + CREATE TABLE {table_uuid} ( a TEXT, b TEXT, @@ -4293,111 +5431,130 @@ def test_xsqlite_execute_closed_connection(): cur.execute(create_sql) with sql.pandasSQL_builder(conn) as pandas_sql: - pandas_sql.execute("INSERT INTO test VALUES('foo', 'bar', 1.234)") + pandas_sql.execute( + f"INSERT INTO {table_uuid} VALUES('foo', 'bar', 1.234)" + ) msg = "Cannot operate on a closed database." with pytest.raises(sqlite3.ProgrammingError, match=msg): - tquery("select * from test", con=conn) + tquery(f"select * from {table_uuid}", con=conn) -def test_xsqlite_keyword_as_column_names(sqlite_buildin): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(["sqlite_buildin"], table_uuid="test_xsqlite_keyword_as_column_names"), + indirect=True, +) +def test_xsqlite_keyword_as_column_names(connect_and_uuid): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] df = DataFrame({"From": np.ones(5)}) - assert sql.to_sql(df, con=sqlite_buildin, name="testkeywords", index=False) == 5 + assert sql.to_sql(df, con=conn, name=table_uuid, index=False) == 5 -def test_xsqlite_onecolumn_of_integer(sqlite_buildin): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(["sqlite_buildin"], table_uuid="test_xsqlite_onecolumn_of_integer"), + indirect=True, +) +def test_xsqlite_onecolumn_of_integer(connect_and_uuid): # GH 3628 # a column_of_integers dataframe should transfer well to sql - + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] mono_df = DataFrame([1, 2], columns=["c0"]) - assert sql.to_sql(mono_df, con=sqlite_buildin, name="mono_df", index=False) == 2 + assert sql.to_sql(mono_df, con=conn, name=table_uuid, index=False) == 2 # computing the sum via sql - con_x = sqlite_buildin - the_sum = sum(my_c0[0] for my_c0 in con_x.execute("select * from mono_df")) + the_sum = sum(my_c0[0] for my_c0 in conn.execute(f"select * from {table_uuid}")) # it should not fail, and gives 3 ( Issue #3628 ) assert the_sum == 3 - result = sql.read_sql("select * from mono_df", con_x) + result = sql.read_sql(f"select * from {table_uuid}", conn) tm.assert_frame_equal(result, mono_df) -def test_xsqlite_if_exists(sqlite_buildin): +@pytest.mark.parametrize( + "connect_and_uuid", + setup(["sqlite_buildin"], table_uuid="test_xsqlite_if_exists"), + indirect=True, +) +def test_xsqlite_if_exists(connect_and_uuid): + conn = connect_and_uuid["conn"] + table_uuid = connect_and_uuid["table_uuid"] + df_if_exists_1 = DataFrame({"col1": [1, 2], "col2": ["A", "B"]}) df_if_exists_2 = DataFrame({"col1": [3, 4, 5], "col2": ["C", "D", "E"]}) - table_name = "table_if_exists" - sql_select = f"SELECT * FROM {table_name}" + sql_select = f"SELECT * FROM {table_uuid}" msg = "'notvalidvalue' is not valid for if_exists" with pytest.raises(ValueError, match=msg): sql.to_sql( frame=df_if_exists_1, - con=sqlite_buildin, - name=table_name, + con=conn, + name=table_uuid, if_exists="notvalidvalue", ) - drop_table(table_name, sqlite_buildin) + drop_table(table_uuid, conn) # test if_exists='fail' - sql.to_sql( - frame=df_if_exists_1, con=sqlite_buildin, name=table_name, if_exists="fail" - ) - msg = "Table 'table_if_exists' already exists" + sql.to_sql(frame=df_if_exists_1, con=conn, name=table_uuid, if_exists="fail") + msg = f"Table '{table_uuid}' already exists" with pytest.raises(ValueError, match=msg): sql.to_sql( frame=df_if_exists_1, - con=sqlite_buildin, - name=table_name, + con=conn, + name=table_uuid, if_exists="fail", ) # test if_exists='replace' sql.to_sql( frame=df_if_exists_1, - con=sqlite_buildin, - name=table_name, + con=conn, + name=table_uuid, if_exists="replace", index=False, ) - assert tquery(sql_select, con=sqlite_buildin) == [(1, "A"), (2, "B")] + assert tquery(sql_select, con=conn) == [(1, "A"), (2, "B")] assert ( sql.to_sql( frame=df_if_exists_2, - con=sqlite_buildin, - name=table_name, + con=conn, + name=table_uuid, if_exists="replace", index=False, ) == 3 ) - assert tquery(sql_select, con=sqlite_buildin) == [(3, "C"), (4, "D"), (5, "E")] - drop_table(table_name, sqlite_buildin) + assert tquery(sql_select, con=conn) == [(3, "C"), (4, "D"), (5, "E")] + drop_table(table_uuid, conn) # test if_exists='append' assert ( sql.to_sql( frame=df_if_exists_1, - con=sqlite_buildin, - name=table_name, + con=conn, + name=table_uuid, if_exists="fail", index=False, ) == 2 ) - assert tquery(sql_select, con=sqlite_buildin) == [(1, "A"), (2, "B")] + assert tquery(sql_select, con=conn) == [(1, "A"), (2, "B")] assert ( sql.to_sql( frame=df_if_exists_2, - con=sqlite_buildin, - name=table_name, + con=conn, + name=table_uuid, if_exists="append", index=False, ) == 3 ) - assert tquery(sql_select, con=sqlite_buildin) == [ + assert tquery(sql_select, con=conn) == [ (1, "A"), (2, "B"), (3, "C"), (4, "D"), (5, "E"), ] - drop_table(table_name, sqlite_buildin) + drop_table(table_uuid, conn)