Skip to content

[SQL][TESTS][SPARK-52918] Batch JDBC database statements in JDBC suites #51616

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 15 commits into from
+162 −194
Closed

[SQL][TESTS][SPARK-52918] Batch JDBC database statements in JDBC suites #51616

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
635d110
improve JDBCV2Suite.scala
alekjarmov Jul 22, 2025
4a3a820
improve jdbcsuite
alekjarmov Jul 22, 2025
a484afe
Update sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.…
Jul 22, 2025
7eb3d99
scala style fix
alekjarmov Jul 22, 2025
1fcaabe
fix scalastyle
alekjarmov Jul 22, 2025
3803218
use batch statement in JDBCSuite
alekjarmov Jul 22, 2025
11a3085
move jdbcv2suite to batch
alekjarmov Jul 22, 2025
7f42972
split batch further
alekjarmov Jul 22, 2025
e0a0f04
more scalastyle fixes
alekjarmov Jul 22, 2025
0094a96
Batch creates in v2
alekjarmov Jul 22, 2025
190e61c
even more batching in v1
alekjarmov Jul 22, 2025
e162968
scalastyle
alekjarmov Jul 22, 2025
23420cc
add empty lines for readibility
alekjarmov Jul 22, 2025
5cf2c8c
use single batch for insert and create
alekjarmov Jul 23, 2025
3497bf1
group inserts and creates
alekjarmov Jul 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 32 additions & 52 deletions sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,11 +105,9 @@ class JDBCSuite extends QueryTest with SharedSparkSession {
conn.prepareStatement("create schema test").executeUpdate()
conn.prepareStatement(
"create table test.people (name TEXT(32) NOT NULL, theid INTEGER NOT NULL)").executeUpdate()
conn.prepareStatement("insert into test.people values ('fred', 1)").executeUpdate()
conn.prepareStatement("insert into test.people values ('mary', 2)").executeUpdate()
conn.prepareStatement(
"insert into test.people values ('joe ''foo'' \"bar\"', 3)").executeUpdate()
conn.commit()

conn.prepareStatement("insert into test.people values " +
"('fred', 1), ('mary', 2), ('joe ''foo'' \"bar\"', 3)").executeUpdate()

sql(
s"""
Expand Down Expand Up @@ -145,11 +143,10 @@ class JDBCSuite extends QueryTest with SharedSparkSession {

conn.prepareStatement("create table test.inttypes (a INT, b BOOLEAN, c TINYINT, "
+ "d SMALLINT, e BIGINT)").executeUpdate()
conn.prepareStatement("insert into test.inttypes values (1, false, 3, 4, 1234567890123)"
).executeUpdate()
conn.prepareStatement("insert into test.inttypes values (null, null, null, null, null)"
).executeUpdate()
conn.commit()

conn.prepareStatement("insert into test.inttypes values " +
"(1, false, 3, 4, 1234567890123), (null, null, null, null, null)").executeUpdate()

sql(
s"""
|CREATE OR REPLACE TEMPORARY VIEW inttypes
Expand All @@ -176,11 +173,11 @@ class JDBCSuite extends QueryTest with SharedSparkSession {

conn.prepareStatement("create table test.timetypes (a TIME, b DATE, c TIMESTAMP(7))"
).executeUpdate()
conn.prepareStatement("insert into test.timetypes values ('12:34:56', "
+ "'1996-01-01', '2002-02-20 11:22:33.543543543')").executeUpdate()
conn.prepareStatement("insert into test.timetypes values ('12:34:56', "
+ "null, '2002-02-20 11:22:33.543543543')").executeUpdate()
conn.commit()

conn.prepareStatement("insert into test.timetypes values " +
"('12:34:56', '1996-01-01', '2002-02-20 11:22:33.543543543'), " +
"('12:34:56', null, '2002-02-20 11:22:33.543543543')").executeUpdate()

sql(
s"""
|CREATE OR REPLACE TEMPORARY VIEW timetypes
Expand All @@ -191,20 +188,18 @@ class JDBCSuite extends QueryTest with SharedSparkSession {
conn.prepareStatement("CREATE TABLE test.timezone (tz TIMESTAMP WITH TIME ZONE) " +
"AS SELECT '1999-01-08 04:05:06.543543543-08:00'")
.executeUpdate()
conn.commit()

conn.prepareStatement("CREATE TABLE test.array_table (ar Integer ARRAY) " +
"AS SELECT ARRAY[1, 2, 3]")
.executeUpdate()
conn.commit()

conn.prepareStatement("create table test.flttypes (a DOUBLE, b REAL, c DECIMAL(38, 18))"
).executeUpdate()
conn.prepareStatement("insert into test.flttypes values ("
+ "1.0000000000000002220446049250313080847263336181640625, "
+ "1.00000011920928955078125, "
+ "123456789012345.543215432154321)").executeUpdate()
conn.commit()

sql(
s"""
|CREATE OR REPLACE TEMPORARY VIEW flttypes
Expand All @@ -221,7 +216,7 @@ class JDBCSuite extends QueryTest with SharedSparkSession {
conn.prepareStatement("insert into test.nulltypes values ("
+ "null, null, null, null, null, null, null, null, null, "
+ "null, null, null, null, null, null)").executeUpdate()
conn.commit()

sql(
s"""
|CREATE OR REPLACE TEMPORARY VIEW nulltypes
Expand All @@ -232,25 +227,15 @@ class JDBCSuite extends QueryTest with SharedSparkSession {
conn.prepareStatement(
"create table test.emp(name TEXT(32) NOT NULL," +
" theid INTEGER, \"Dept\" INTEGER)").executeUpdate()
conn.prepareStatement(
"insert into test.emp values ('fred', 1, 10)").executeUpdate()
conn.prepareStatement(
"insert into test.emp values ('mary', 2, null)").executeUpdate()
conn.prepareStatement(
"insert into test.emp values ('joe ''foo'' \"bar\"', 3, 30)").executeUpdate()
conn.prepareStatement(
"insert into test.emp values ('kathy', null, null)").executeUpdate()
conn.commit()

conn.prepareStatement("insert into test.emp values " +
"('fred', 1, 10), ('mary', 2, null), ('joe ''foo'' \"bar\"', 3, 30), ('kathy', null, null)").executeUpdate()

conn.prepareStatement(
"create table test.seq(id INTEGER)").executeUpdate()
(0 to 6).foreach { value =>
conn.prepareStatement(
s"insert into test.seq values ($value)").executeUpdate()
}
conn.prepareStatement(
"insert into test.seq values (null)").executeUpdate()
conn.commit()

val seqValues = (0 to 6).map(value => s"($value)").mkString(", ") + ", (null)"
conn.prepareStatement(s"insert into test.seq values $seqValues").executeUpdate()

sql(
s"""
Expand All @@ -263,10 +248,9 @@ class JDBCSuite extends QueryTest with SharedSparkSession {
conn.prepareStatement(
"""create table test."mixedCaseCols" ("Name" TEXT(32), "Id" INTEGER NOT NULL)""")
.executeUpdate()
conn.prepareStatement("""insert into test."mixedCaseCols" values ('fred', 1)""").executeUpdate()
conn.prepareStatement("""insert into test."mixedCaseCols" values ('mary', 2)""").executeUpdate()
conn.prepareStatement("""insert into test."mixedCaseCols" values (null, 3)""").executeUpdate()
conn.commit()

conn.prepareStatement("""insert into test."mixedCaseCols" values """ +
"""('fred', 1), ('mary', 2), (null, 3)""").executeUpdate()

sql(
s"""
Expand All @@ -278,25 +262,21 @@ class JDBCSuite extends QueryTest with SharedSparkSession {
conn.prepareStatement("CREATE TABLE test.partition (THEID INTEGER, `THE ID` INTEGER) " +
"AS SELECT 1, 1")
.executeUpdate()
conn.commit()

conn.prepareStatement("CREATE TABLE test.datetime (d DATE, t TIMESTAMP)").executeUpdate()
conn.prepareStatement(
"INSERT INTO test.datetime VALUES ('2018-07-06', '2018-07-06 05:50:00.0')").executeUpdate()
conn.prepareStatement(
"INSERT INTO test.datetime VALUES ('2018-07-06', '2018-07-06 08:10:08.0')").executeUpdate()
conn.prepareStatement(
"INSERT INTO test.datetime VALUES ('2018-07-08', '2018-07-08 13:32:01.0')").executeUpdate()
conn.prepareStatement(
"INSERT INTO test.datetime VALUES ('2018-07-12', '2018-07-12 09:51:15.0')").executeUpdate()
conn.commit()

conn.prepareStatement("INSERT INTO test.datetime VALUES " +
"('2018-07-06', '2018-07-06 05:50:00.0'), " +
"('2018-07-06', '2018-07-06 08:10:08.0'), " +
"('2018-07-08', '2018-07-08 13:32:01.0'), " +
"('2018-07-12', '2018-07-12 09:51:15.0')").executeUpdate()

conn.prepareStatement(
"CREATE TABLE test.composite_name (`last name` TEXT(32) NOT NULL, id INTEGER NOT NULL)")
.executeUpdate()
conn.prepareStatement("INSERT INTO test.composite_name VALUES ('smith', 1)").executeUpdate()
conn.prepareStatement("INSERT INTO test.composite_name VALUES ('jones', 2)").executeUpdate()
conn.commit()

conn.prepareStatement("INSERT INTO test.composite_name VALUES " +
"('smith', 1), ('jones', 2)").executeUpdate()

sql(
s"""
Expand Down
86 changes: 39 additions & 47 deletions sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,33 +164,35 @@ class JDBCV2Suite extends QueryTest with SharedSparkSession with ExplainSuiteHel
conn.prepareStatement(
"CREATE TABLE \"test\".\"people\" (name TEXT(32) NOT NULL, id INTEGER NOT NULL)")
.executeUpdate()
conn.prepareStatement("INSERT INTO \"test\".\"people\" VALUES ('fred', 1)").executeUpdate()
conn.prepareStatement("INSERT INTO \"test\".\"people\" VALUES ('mary', 2)").executeUpdate()

conn.prepareStatement("INSERT INTO \"test\".\"people\" VALUES " +
"('fred', 1), ('mary', 2)").executeUpdate()

conn.prepareStatement(
"CREATE TABLE \"test\".\"employee\" (dept INTEGER, name TEXT(32), salary NUMERIC(20, 2)," +
" bonus DOUBLE, is_manager BOOLEAN)").executeUpdate()
conn.prepareStatement(
"INSERT INTO \"test\".\"employee\" VALUES (1, 'amy', 10000, 1000, true)").executeUpdate()
conn.prepareStatement(
"INSERT INTO \"test\".\"employee\" VALUES (2, 'alex', 12000, 1200, false)").executeUpdate()
conn.prepareStatement(
"INSERT INTO \"test\".\"employee\" VALUES (1, 'cathy', 9000, 1200, false)").executeUpdate()
conn.prepareStatement(
"INSERT INTO \"test\".\"employee\" VALUES (2, 'david', 10000, 1300, true)").executeUpdate()
conn.prepareStatement(
"INSERT INTO \"test\".\"employee\" VALUES (6, 'jen', 12000, 1200, true)").executeUpdate()

conn.prepareStatement("INSERT INTO \"test\".\"employee\" VALUES " +
"(1, 'amy', 10000, 1000, true), " +
"(2, 'alex', 12000, 1200, false), " +
"(1, 'cathy', 9000, 1200, false), " +
"(2, 'david', 10000, 1300, true), " +
"(6, 'jen', 12000, 1200, true)").executeUpdate()

conn.prepareStatement(
"CREATE TABLE \"test\".\"dept\" (\"dept id\" INTEGER NOT NULL, \"dept.id\" INTEGER)")
.executeUpdate()
conn.prepareStatement("INSERT INTO \"test\".\"dept\" VALUES (1, 1)").executeUpdate()
conn.prepareStatement("INSERT INTO \"test\".\"dept\" VALUES (2, 1)").executeUpdate()

// scalastyle:off
conn.prepareStatement(
"CREATE TABLE \"test\".\"person\" (\"名\" INTEGER NOT NULL)").executeUpdate()
// scalastyle:on
conn.prepareStatement("INSERT INTO \"test\".\"person\" VALUES (1)").executeUpdate()
conn.prepareStatement("INSERT INTO \"test\".\"person\" VALUES (2)").executeUpdate()
conn.prepareStatement("INSERT INTO \"test\".\"dept\" VALUES " +
"(1, 1), (2, 1)").executeUpdate()

conn
.prepareStatement("CREATE TABLE \"test\".\"person\" (\"名\" INTEGER NOT NULL)")
.executeUpdate()

// Optimized: Batch insert for person table (was 2 separate INSERTs)
conn.prepareStatement("INSERT INTO \"test\".\"person\" VALUES (1), (2)").executeUpdate()

conn.prepareStatement(
"""CREATE TABLE "test"."view1" ("|col1" INTEGER, "|col2" INTEGER)""").executeUpdate()
conn.prepareStatement(
Expand All @@ -199,30 +201,27 @@ class JDBCV2Suite extends QueryTest with SharedSparkSession with ExplainSuiteHel
conn.prepareStatement(
"CREATE TABLE \"test\".\"item\" (id INTEGER, name TEXT(32), price NUMERIC(23, 3))")
.executeUpdate()

conn.prepareStatement("INSERT INTO \"test\".\"item\" VALUES " +
"(1, 'bottle', 11111111111111111111.123)").executeUpdate()
conn.prepareStatement("INSERT INTO \"test\".\"item\" VALUES " +
"(1, 'bottle', 11111111111111111111.123), " +
"(1, 'bottle', 99999999999999999999.123)").executeUpdate()

conn.prepareStatement(
"CREATE TABLE \"test\".\"datetime\" (name TEXT(32), date1 DATE, time1 TIMESTAMP)")
.executeUpdate()

conn.prepareStatement("INSERT INTO \"test\".\"datetime\" VALUES " +
"('amy', '2022-05-19', '2022-05-19 00:00:00')").executeUpdate()
conn.prepareStatement("INSERT INTO \"test\".\"datetime\" VALUES " +
"('amy', '2022-05-19', '2022-05-19 00:00:00'), " +
"('alex', '2022-05-18', '2022-05-18 00:00:00')").executeUpdate()

conn.prepareStatement(
"CREATE TABLE \"test\".\"address\" (email TEXT(32) NOT NULL)").executeUpdate()

conn.prepareStatement("INSERT INTO \"test\".\"address\" VALUES " +
"('abc_def@gmail.com')").executeUpdate()
conn.prepareStatement("INSERT INTO \"test\".\"address\" VALUES " +
"('abc%def@gmail.com')").executeUpdate()
conn.prepareStatement("INSERT INTO \"test\".\"address\" VALUES " +
"('abc%_def@gmail.com')").executeUpdate()
conn.prepareStatement("INSERT INTO \"test\".\"address\" VALUES " +
"('abc_%def@gmail.com')").executeUpdate()
conn.prepareStatement("INSERT INTO \"test\".\"address\" VALUES " +
"('abc_def@gmail.com'), " +
"('abc%def@gmail.com'), " +
"('abc%_def@gmail.com'), " +
"('abc_%def@gmail.com'), " +
"('abc_''%def@gmail.com')").executeUpdate()

conn.prepareStatement("CREATE TABLE \"test\".\"binary_tab\" (name TEXT(32),b BINARY(20))")
Expand All @@ -234,25 +233,18 @@ class JDBCV2Suite extends QueryTest with SharedSparkSession with ExplainSuiteHel

conn.prepareStatement("CREATE TABLE \"test\".\"employee_bonus\" " +
"(name TEXT(32), salary NUMERIC(20, 2), bonus DOUBLE, factor DOUBLE)").executeUpdate()
conn.prepareStatement("INSERT INTO \"test\".\"employee_bonus\" " +
"VALUES ('amy', 10000, 1000, 0.1)").executeUpdate()
conn.prepareStatement("INSERT INTO \"test\".\"employee_bonus\" " +
"VALUES ('alex', 12000, 1200, 0.1)").executeUpdate()
conn.prepareStatement("INSERT INTO \"test\".\"employee_bonus\" " +
"VALUES ('cathy', 8000, 1200, 0.15)").executeUpdate()
conn.prepareStatement("INSERT INTO \"test\".\"employee_bonus\" " +
"VALUES ('david', 10000, 1300, 0.13)").executeUpdate()
conn.prepareStatement("INSERT INTO \"test\".\"employee_bonus\" " +
"VALUES ('jen', 12000, 2400, 0.2)").executeUpdate()

conn.prepareStatement("INSERT INTO \"test\".\"employee_bonus\" VALUES " +
"('amy', 10000, 1000, 0.1), " +
"('alex', 12000, 1200, 0.1), " +
"('cathy', 8000, 1200, 0.15), " +
"('david', 10000, 1300, 0.13), " +
"('jen', 12000, 2400, 0.2)").executeUpdate()

conn.prepareStatement(
"CREATE TABLE \"test\".\"strings_with_nulls\" (str TEXT(32))").executeUpdate()
conn.prepareStatement("INSERT INTO \"test\".\"strings_with_nulls\" VALUES " +
"('abc')").executeUpdate()
conn.prepareStatement("INSERT INTO \"test\".\"strings_with_nulls\" VALUES " +
"('a a a')").executeUpdate()
conn.prepareStatement("INSERT INTO \"test\".\"strings_with_nulls\" VALUES " +
"(null)").executeUpdate()
"('abc'), ('a a a'), (null)").executeUpdate()
}
h2Dialect.registerFunction("my_avg", IntegralAverage)
h2Dialect.registerFunction("my_strlen", StrLen(CharLength))
Expand Down