From fc62886d4c6636c275d652cbb4cd7049ed58c372 Mon Sep 17 00:00:00 2001 From: Szehon Ho Date: Wed, 4 Jun 2025 16:57:51 -0700 Subject: [PATCH 1/6] [SPARK-52235][SQL][FOLLOW-UP] Add ANSI mode check for new tests --- .../DataSourceV2DataFrameSuite.scala | 177 +++++++++--------- 1 file changed, 90 insertions(+), 87 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala index a7bfcfcfea62b..fe9117835ed48 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala @@ -676,104 +676,107 @@ class DataSourceV2DataFrameSuite test("create/replace table default value expression should have a cast") { val tableName = "testcat.ns1.ns2.tbl" - withTable(tableName) { - - val createExec = executeAndKeepPhysicalPlan[CreateTableExec] { - sql( - s""" - |CREATE TABLE $tableName ( - | col1 int, - | col2 timestamp DEFAULT '2018-11-17 13:33:33', - | col3 double DEFAULT 1) - |""".stripMargin) - } - checkDefaultValues( - createExec.columns, - Array( - null, - new ColumnDefaultValue( - "'2018-11-17 13:33:33'", - new LiteralValue(1542490413000000L, TimestampType), - new LiteralValue(1542490413000000L, TimestampType)), - new ColumnDefaultValue( - "1", - new V2Cast(LiteralValue(1, IntegerType), IntegerType, DoubleType), - LiteralValue(1.0, DoubleType)))) + withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { + withTable(tableName) { + val createExec = executeAndKeepPhysicalPlan[CreateTableExec] { + sql( + s""" + |CREATE TABLE $tableName ( + | col1 int, + | col2 timestamp DEFAULT '2018-11-17 13:33:33', + | col3 double DEFAULT 1) + |""".stripMargin) + } + checkDefaultValues( + createExec.columns, + Array( + null, + new ColumnDefaultValue( + "'2018-11-17 13:33:33'", + new LiteralValue(1542490413000000L, TimestampType), + new LiteralValue(1542490413000000L, TimestampType)), + new ColumnDefaultValue( + "1", + new V2Cast(LiteralValue(1, IntegerType), IntegerType, DoubleType), + LiteralValue(1.0, DoubleType)))) - val replaceExec = executeAndKeepPhysicalPlan[ReplaceTableExec] { - sql( - s""" - |REPLACE TABLE $tableName ( - | col1 int, - | col2 timestamp DEFAULT '2022-02-23 05:55:55', - | col3 double DEFAULT (1 + 1)) - |""".stripMargin) + val replaceExec = executeAndKeepPhysicalPlan[ReplaceTableExec] { + sql( + s""" + |REPLACE TABLE $tableName ( + | col1 int, + | col2 timestamp DEFAULT '2022-02-23 05:55:55', + | col3 double DEFAULT (1 + 1)) + |""".stripMargin) + } + checkDefaultValues( + replaceExec.columns, + Array( + null, + new ColumnDefaultValue( + "'2022-02-23 05:55:55'", + LiteralValue(1645624555000000L, TimestampType), + LiteralValue(1645624555000000L, TimestampType)), + new ColumnDefaultValue( + "(1 + 1)", + new V2Cast( + new GeneralScalarExpression("+", Array(LiteralValue(1, IntegerType), + LiteralValue(1, IntegerType))), + IntegerType, + DoubleType), + LiteralValue(2.0, DoubleType)))) } - checkDefaultValues( - replaceExec.columns, - Array( - null, - new ColumnDefaultValue( - "'2022-02-23 05:55:55'", - LiteralValue(1645624555000000L, TimestampType), - LiteralValue(1645624555000000L, TimestampType)), - new ColumnDefaultValue( - "(1 + 1)", - new V2Cast( - new GeneralScalarExpression("+", Array(LiteralValue(1, IntegerType), - LiteralValue(1, IntegerType))), - IntegerType, - DoubleType), - LiteralValue(2.0, DoubleType)))) } } test("alter table default value expression should have a cast") { val tableName = "testcat.ns1.ns2.tbl" - withTable(tableName) { + withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { + withTable(tableName) { - sql(s"CREATE TABLE $tableName (col1 int) using foo") - val alterExec = executeAndKeepPhysicalPlan[AlterTableExec] { - sql( - s""" - |ALTER TABLE $tableName ADD COLUMNS ( - | col2 timestamp DEFAULT '2018-11-17 13:33:33', - | col3 double DEFAULT 1) - |""".stripMargin) - } + sql(s"CREATE TABLE $tableName (col1 int) using foo") + val alterExec = executeAndKeepPhysicalPlan[AlterTableExec] { + sql( + s""" + |ALTER TABLE $tableName ADD COLUMNS ( + | col2 timestamp DEFAULT '2018-11-17 13:33:33', + | col3 double DEFAULT 1) + |""".stripMargin) + } - checkDefaultValues( - alterExec.changes.map(_.asInstanceOf[AddColumn]).toArray, - Array( - new ColumnDefaultValue( - "'2018-11-17 13:33:33'", - LiteralValue(1542490413000000L, TimestampType), - LiteralValue(1542490413000000L, TimestampType)), - new ColumnDefaultValue( - "1", - new V2Cast(LiteralValue(1, IntegerType), IntegerType, DoubleType), - LiteralValue(1.0, DoubleType)))) + checkDefaultValues( + alterExec.changes.map(_.asInstanceOf[AddColumn]).toArray, + Array( + new ColumnDefaultValue( + "'2018-11-17 13:33:33'", + LiteralValue(1542490413000000L, TimestampType), + LiteralValue(1542490413000000L, TimestampType)), + new ColumnDefaultValue( + "1", + new V2Cast(LiteralValue(1, IntegerType), IntegerType, DoubleType), + LiteralValue(1.0, DoubleType)))) - val alterCol1 = executeAndKeepPhysicalPlan[AlterTableExec] { - sql( - s""" - |ALTER TABLE $tableName ALTER COLUMN - | col2 SET DEFAULT '2022-02-23 05:55:55', - | col3 SET DEFAULT (1 + 1) - |""".stripMargin) + val alterCol1 = executeAndKeepPhysicalPlan[AlterTableExec] { + sql( + s""" + |ALTER TABLE $tableName ALTER COLUMN + | col2 SET DEFAULT '2022-02-23 05:55:55', + | col3 SET DEFAULT (1 + 1) + |""".stripMargin) + } + checkDefaultValues( + alterCol1.changes.map(_.asInstanceOf[UpdateColumnDefaultValue]).toArray, + Array( + new DefaultValue("'2022-02-23 05:55:55'", + LiteralValue(1645624555000000L, TimestampType)), + new DefaultValue( + "(1 + 1)", + new V2Cast( + new GeneralScalarExpression("+", Array(LiteralValue(1, IntegerType), + LiteralValue(1, IntegerType))), + IntegerType, + DoubleType)))) } - checkDefaultValues( - alterCol1.changes.map(_.asInstanceOf[UpdateColumnDefaultValue]).toArray, - Array( - new DefaultValue("'2022-02-23 05:55:55'", - LiteralValue(1645624555000000L, TimestampType)), - new DefaultValue( - "(1 + 1)", - new V2Cast( - new GeneralScalarExpression("+", Array(LiteralValue(1, IntegerType), - LiteralValue(1, IntegerType))), - IntegerType, - DoubleType)))) } } From fd3fcaa0c2c11b2717e9f8881312b20ddd2580a0 Mon Sep 17 00:00:00 2001 From: Szehon Ho Date: Thu, 5 Jun 2025 12:15:57 -0700 Subject: [PATCH 2/6] Disable ANSI for all test in suite --- .../DataSourceV2DataFrameSuite.scala | 408 +++++++++--------- 1 file changed, 200 insertions(+), 208 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala index fe9117835ed48..92e0485b05a88 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala @@ -43,6 +43,7 @@ class DataSourceV2DataFrameSuite before { spark.conf.set("spark.sql.catalog.testcat", classOf[InMemoryTableCatalog].getName) spark.conf.set("spark.sql.catalog.testcat2", classOf[InMemoryTableCatalog].getName) + spark.conf.set(SQLConf.ANSI_ENABLED.key, "true") } after { @@ -352,185 +353,180 @@ class DataSourceV2DataFrameSuite test("create/replace table with complex foldable default values") { val tableName = "testcat.ns1.ns2.tbl" withTable(tableName) { - withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { - val createExec = executeAndKeepPhysicalPlan[CreateTableExec] { - sql( - s""" - |CREATE TABLE $tableName ( - | id INT, - | salary INT DEFAULT (100 + 23), - | dep STRING DEFAULT ('h' || 'r'), - | active BOOLEAN DEFAULT CAST(1 AS BOOLEAN) - |) USING foo - |""".stripMargin) - } + val createExec = executeAndKeepPhysicalPlan[CreateTableExec] { + sql( + s""" + |CREATE TABLE $tableName ( + | id INT, + | salary INT DEFAULT (100 + 23), + | dep STRING DEFAULT ('h' || 'r'), + | active BOOLEAN DEFAULT CAST(1 AS BOOLEAN) + |) USING foo + |""".stripMargin) + } - checkDefaultValues( - createExec.columns, - Array( - null, - new ColumnDefaultValue( - "(100 + 23)", - new GeneralScalarExpression( - "+", - Array(LiteralValue(100, IntegerType), LiteralValue(23, IntegerType))), - LiteralValue(123, IntegerType)), - new ColumnDefaultValue( - "('h' || 'r')", - new GeneralScalarExpression( - "CONCAT", - Array( - LiteralValue(UTF8String.fromString("h"), StringType), - LiteralValue(UTF8String.fromString("r"), StringType))), - LiteralValue(UTF8String.fromString("hr"), StringType)), - new ColumnDefaultValue( - "CAST(1 AS BOOLEAN)", - new V2Cast(LiteralValue(1, IntegerType), IntegerType, BooleanType), - LiteralValue(true, BooleanType)))) + checkDefaultValues( + createExec.columns, + Array( + null, + new ColumnDefaultValue( + "(100 + 23)", + new GeneralScalarExpression( + "+", + Array(LiteralValue(100, IntegerType), LiteralValue(23, IntegerType))), + LiteralValue(123, IntegerType)), + new ColumnDefaultValue( + "('h' || 'r')", + new GeneralScalarExpression( + "CONCAT", + Array( + LiteralValue(UTF8String.fromString("h"), StringType), + LiteralValue(UTF8String.fromString("r"), StringType))), + LiteralValue(UTF8String.fromString("hr"), StringType)), + new ColumnDefaultValue( + "CAST(1 AS BOOLEAN)", + new V2Cast(LiteralValue(1, IntegerType), IntegerType, BooleanType), + LiteralValue(true, BooleanType)))) - val df1 = Seq(1).toDF("id") - df1.writeTo(tableName).append() + val df1 = Seq(1).toDF("id") + df1.writeTo(tableName).append() - sql(s"ALTER TABLE $tableName ALTER COLUMN dep SET DEFAULT ('i' || 't')") + sql(s"ALTER TABLE $tableName ALTER COLUMN dep SET DEFAULT ('i' || 't')") - val df2 = Seq(2).toDF("id") - df2.writeTo(tableName).append() + val df2 = Seq(2).toDF("id") + df2.writeTo(tableName).append() - checkAnswer( - sql(s"SELECT * FROM $tableName"), - Seq( - Row(1, 123, "hr", true), - Row(2, 123, "it", true))) + checkAnswer( + sql(s"SELECT * FROM $tableName"), + Seq( + Row(1, 123, "hr", true), + Row(2, 123, "it", true))) - val replaceExec = executeAndKeepPhysicalPlan[ReplaceTableExec] { - sql( - s""" - |REPLACE TABLE $tableName ( - | id INT, - | salary INT DEFAULT (50 * 2), - | dep STRING DEFAULT ('un' || 'known'), - | active BOOLEAN DEFAULT CAST(0 AS BOOLEAN) - |) USING foo - |""".stripMargin) - } + val replaceExec = executeAndKeepPhysicalPlan[ReplaceTableExec] { + sql( + s""" + |REPLACE TABLE $tableName ( + | id INT, + | salary INT DEFAULT (50 * 2), + | dep STRING DEFAULT ('un' || 'known'), + | active BOOLEAN DEFAULT CAST(0 AS BOOLEAN) + |) USING foo + |""".stripMargin) + } - checkDefaultValues( - replaceExec.columns, - Array( - null, - new ColumnDefaultValue( - "(50 * 2)", - new GeneralScalarExpression( - "*", - Array(LiteralValue(50, IntegerType), LiteralValue(2, IntegerType))), - LiteralValue(100, IntegerType)), - new ColumnDefaultValue( - "('un' || 'known')", - new GeneralScalarExpression( - "CONCAT", - Array( - LiteralValue(UTF8String.fromString("un"), StringType), - LiteralValue(UTF8String.fromString("known"), StringType))), - LiteralValue(UTF8String.fromString("unknown"), StringType)), - new ColumnDefaultValue( - "CAST(0 AS BOOLEAN)", - new V2Cast(LiteralValue(0, IntegerType), IntegerType, BooleanType), - LiteralValue(false, BooleanType)))) + checkDefaultValues( + replaceExec.columns, + Array( + null, + new ColumnDefaultValue( + "(50 * 2)", + new GeneralScalarExpression( + "*", + Array(LiteralValue(50, IntegerType), LiteralValue(2, IntegerType))), + LiteralValue(100, IntegerType)), + new ColumnDefaultValue( + "('un' || 'known')", + new GeneralScalarExpression( + "CONCAT", + Array( + LiteralValue(UTF8String.fromString("un"), StringType), + LiteralValue(UTF8String.fromString("known"), StringType))), + LiteralValue(UTF8String.fromString("unknown"), StringType)), + new ColumnDefaultValue( + "CAST(0 AS BOOLEAN)", + new V2Cast(LiteralValue(0, IntegerType), IntegerType, BooleanType), + LiteralValue(false, BooleanType)))) - val df3 = Seq(1).toDF("id") - df3.writeTo(tableName).append() + val df3 = Seq(1).toDF("id") + df3.writeTo(tableName).append() - checkAnswer( - sql(s"SELECT * FROM $tableName"), - Seq(Row(1, 100, "unknown", false))) - } + checkAnswer( + sql(s"SELECT * FROM $tableName"), + Seq(Row(1, 100, "unknown", false))) } } + test("alter table add column with complex foldable default values") { val tableName = "testcat.ns1.ns2.tbl" - withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { - withTable(tableName) { - sql( - s""" - |CREATE TABLE $tableName ( - | dummy INT - |) USING foo - |""".stripMargin) - - val alterExec = executeAndKeepPhysicalPlan[AlterTableExec] { - sql(s"ALTER TABLE $tableName ADD COLUMNS (" + - s"salary INT DEFAULT (100 + 23), " + - s"dep STRING DEFAULT ('h' || 'r'), " + - s"active BOOLEAN DEFAULT CAST(1 AS BOOLEAN))") - } + withTable(tableName) { + sql( + s""" + |CREATE TABLE $tableName ( + | dummy INT + |) USING foo + |""".stripMargin) - checkDefaultValues( - alterExec.changes.map(_.asInstanceOf[AddColumn]).toArray, - Array( - new ColumnDefaultValue( - "(100 + 23)", - new GeneralScalarExpression( - "+", - Array(LiteralValue(100, IntegerType), LiteralValue(23, IntegerType))), - LiteralValue(123, IntegerType)), - new ColumnDefaultValue( - "('h' || 'r')", - new GeneralScalarExpression( - "CONCAT", - Array( - LiteralValue(UTF8String.fromString("h"), StringType), - LiteralValue(UTF8String.fromString("r"), StringType))), - LiteralValue(UTF8String.fromString("hr"), StringType)), - new ColumnDefaultValue( - "CAST(1 AS BOOLEAN)", - new V2Cast(LiteralValue(1, IntegerType), IntegerType, BooleanType), - LiteralValue(true, BooleanType)))) + val alterExec = executeAndKeepPhysicalPlan[AlterTableExec] { + sql(s"ALTER TABLE $tableName ADD COLUMNS (" + + s"salary INT DEFAULT (100 + 23), " + + s"dep STRING DEFAULT ('h' || 'r'), " + + s"active BOOLEAN DEFAULT CAST(1 AS BOOLEAN))") } + + checkDefaultValues( + alterExec.changes.map(_.asInstanceOf[AddColumn]).toArray, + Array( + new ColumnDefaultValue( + "(100 + 23)", + new GeneralScalarExpression( + "+", + Array(LiteralValue(100, IntegerType), LiteralValue(23, IntegerType))), + LiteralValue(123, IntegerType)), + new ColumnDefaultValue( + "('h' || 'r')", + new GeneralScalarExpression( + "CONCAT", + Array( + LiteralValue(UTF8String.fromString("h"), StringType), + LiteralValue(UTF8String.fromString("r"), StringType))), + LiteralValue(UTF8String.fromString("hr"), StringType)), + new ColumnDefaultValue( + "CAST(1 AS BOOLEAN)", + new V2Cast(LiteralValue(1, IntegerType), IntegerType, BooleanType), + LiteralValue(true, BooleanType)))) } } test("alter table alter column with complex foldable default values") { val tableName = "testcat.ns1.ns2.tbl" - withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { - withTable(tableName) { - sql( - s""" - |CREATE TABLE $tableName ( - | salary INT DEFAULT (100 + 23), - | dep STRING DEFAULT ('h' || 'r'), - | active BOOLEAN DEFAULT CAST(1 AS BOOLEAN) - |) USING foo - |""".stripMargin) - - val alterExecCol1 = executeAndKeepPhysicalPlan[AlterTableExec] { - sql( - s""" - |ALTER TABLE $tableName ALTER COLUMN - | salary SET DEFAULT (123 + 56), - | dep SET DEFAULT ('r' || 'l'), - | active SET DEFAULT CAST(0 AS BOOLEAN) - |""".stripMargin) - } - checkDefaultValues( - alterExecCol1.changes.map(_.asInstanceOf[UpdateColumnDefaultValue]).toArray, - Array( - new DefaultValue( - "(123 + 56)", - new GeneralScalarExpression( - "+", - Array(LiteralValue(123, IntegerType), LiteralValue(56, IntegerType)))), - new DefaultValue( - "('r' || 'l')", - new GeneralScalarExpression( - "CONCAT", - Array( - LiteralValue(UTF8String.fromString("r"), StringType), - LiteralValue(UTF8String.fromString("l"), StringType)))), - new DefaultValue( - "CAST(0 AS BOOLEAN)", - new V2Cast(LiteralValue(0, IntegerType), IntegerType, BooleanType)))) + withTable(tableName) { + sql( + s""" + |CREATE TABLE $tableName ( + | salary INT DEFAULT (100 + 23), + | dep STRING DEFAULT ('h' || 'r'), + | active BOOLEAN DEFAULT CAST(1 AS BOOLEAN) + |) USING foo + |""".stripMargin) + + val alterExecCol1 = executeAndKeepPhysicalPlan[AlterTableExec] { + sql( + s""" + |ALTER TABLE $tableName ALTER COLUMN + | salary SET DEFAULT (123 + 56), + | dep SET DEFAULT ('r' || 'l'), + | active SET DEFAULT CAST(0 AS BOOLEAN) + |""".stripMargin) } + checkDefaultValues( + alterExecCol1.changes.map(_.asInstanceOf[UpdateColumnDefaultValue]).toArray, + Array( + new DefaultValue( + "(123 + 56)", + new GeneralScalarExpression( + "+", + Array(LiteralValue(123, IntegerType), LiteralValue(56, IntegerType)))), + new DefaultValue( + "('r' || 'l')", + new GeneralScalarExpression( + "CONCAT", + Array( + LiteralValue(UTF8String.fromString("r"), StringType), + LiteralValue(UTF8String.fromString("l"), StringType)))), + new DefaultValue( + "CAST(0 AS BOOLEAN)", + new V2Cast(LiteralValue(0, IntegerType), IntegerType, BooleanType)))) } } @@ -676,64 +672,61 @@ class DataSourceV2DataFrameSuite test("create/replace table default value expression should have a cast") { val tableName = "testcat.ns1.ns2.tbl" - withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { - withTable(tableName) { - val createExec = executeAndKeepPhysicalPlan[CreateTableExec] { - sql( - s""" - |CREATE TABLE $tableName ( - | col1 int, - | col2 timestamp DEFAULT '2018-11-17 13:33:33', - | col3 double DEFAULT 1) - |""".stripMargin) - } - checkDefaultValues( - createExec.columns, - Array( - null, - new ColumnDefaultValue( - "'2018-11-17 13:33:33'", - new LiteralValue(1542490413000000L, TimestampType), - new LiteralValue(1542490413000000L, TimestampType)), - new ColumnDefaultValue( - "1", - new V2Cast(LiteralValue(1, IntegerType), IntegerType, DoubleType), - LiteralValue(1.0, DoubleType)))) + withTable(tableName) { + val createExec = executeAndKeepPhysicalPlan[CreateTableExec] { + sql( + s""" + |CREATE TABLE $tableName ( + | col1 int, + | col2 timestamp DEFAULT '2018-11-17 13:33:33', + | col3 double DEFAULT 1) + |""".stripMargin) + } + checkDefaultValues( + createExec.columns, + Array( + null, + new ColumnDefaultValue( + "'2018-11-17 13:33:33'", + LiteralValue(1542490413000000L, TimestampType), + LiteralValue(1542490413000000L, TimestampType)), + new ColumnDefaultValue( + "1", + new V2Cast(LiteralValue(1, IntegerType), IntegerType, DoubleType), + LiteralValue(1.0, DoubleType)))) - val replaceExec = executeAndKeepPhysicalPlan[ReplaceTableExec] { - sql( - s""" - |REPLACE TABLE $tableName ( - | col1 int, - | col2 timestamp DEFAULT '2022-02-23 05:55:55', - | col3 double DEFAULT (1 + 1)) - |""".stripMargin) - } - checkDefaultValues( - replaceExec.columns, - Array( - null, - new ColumnDefaultValue( - "'2022-02-23 05:55:55'", - LiteralValue(1645624555000000L, TimestampType), - LiteralValue(1645624555000000L, TimestampType)), - new ColumnDefaultValue( - "(1 + 1)", - new V2Cast( - new GeneralScalarExpression("+", Array(LiteralValue(1, IntegerType), - LiteralValue(1, IntegerType))), - IntegerType, - DoubleType), - LiteralValue(2.0, DoubleType)))) + val replaceExec = executeAndKeepPhysicalPlan[ReplaceTableExec] { + sql( + s""" + |REPLACE TABLE $tableName ( + | col1 int, + | col2 timestamp DEFAULT '2022-02-23 05:55:55', + | col3 double DEFAULT (1 + 1)) + |""".stripMargin) } + checkDefaultValues( + replaceExec.columns, + Array( + null, + new ColumnDefaultValue( + "'2022-02-23 05:55:55'", + LiteralValue(1645624555000000L, TimestampType), + LiteralValue(1645624555000000L, TimestampType)), + new ColumnDefaultValue( + "(1 + 1)", + new V2Cast( + new GeneralScalarExpression("+", Array(LiteralValue(1, IntegerType), + LiteralValue(1, IntegerType))), + IntegerType, + DoubleType), + LiteralValue(2.0, DoubleType)))) } } + test("alter table default value expression should have a cast") { val tableName = "testcat.ns1.ns2.tbl" - withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { withTable(tableName) { - sql(s"CREATE TABLE $tableName (col1 int) using foo") val alterExec = executeAndKeepPhysicalPlan[AlterTableExec] { sql( @@ -777,7 +770,6 @@ class DataSourceV2DataFrameSuite IntegerType, DoubleType)))) } - } } test("write with supported expression-based default values") { From ac46066c8a21902d1c3d6b9ae38a7c12b4810db7 Mon Sep 17 00:00:00 2001 From: Szehon Ho Date: Thu, 5 Jun 2025 15:04:33 -0700 Subject: [PATCH 3/6] Use override --- .../spark/sql/connector/DataSourceV2DataFrameSuite.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala index 92e0485b05a88..aa9d82b02bf8e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.connector import java.util.Collections -import org.apache.spark.SparkException +import org.apache.spark.{SparkConf, SparkException} import org.apache.spark.sql.{AnalysisException, DataFrame, Row, SaveMode} import org.apache.spark.sql.QueryTest.withQueryExecutionsCaptured import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException @@ -40,10 +40,13 @@ class DataSourceV2DataFrameSuite import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ import testImplicits._ + override protected def sparkConf: SparkConf = + // Disable read-side char padding so that the generated code is less than 8000. + super.sparkConf.set(SQLConf.ANSI_ENABLED, true) + before { spark.conf.set("spark.sql.catalog.testcat", classOf[InMemoryTableCatalog].getName) spark.conf.set("spark.sql.catalog.testcat2", classOf[InMemoryTableCatalog].getName) - spark.conf.set(SQLConf.ANSI_ENABLED.key, "true") } after { From 5ebd6fefbd8a7fa0ba1e1e8e300548ebbd7aec43 Mon Sep 17 00:00:00 2001 From: Szehon Ho Date: Thu, 5 Jun 2025 15:07:04 -0700 Subject: [PATCH 4/6] Update sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala Co-authored-by: Wenchen Fan --- .../apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala index aa9d82b02bf8e..25874149f7a9c 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala @@ -41,7 +41,6 @@ class DataSourceV2DataFrameSuite import testImplicits._ override protected def sparkConf: SparkConf = - // Disable read-side char padding so that the generated code is less than 8000. super.sparkConf.set(SQLConf.ANSI_ENABLED, true) before { From 0927a61b526e2315c36d9eb7a71799502b4375e7 Mon Sep 17 00:00:00 2001 From: Wenchen Fan Date: Thu, 5 Jun 2025 17:00:32 -0700 Subject: [PATCH 5/6] fix --- .../sql/connector/DataSourceV2DataFrameSuite.scala | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala index 25874149f7a9c..342eefa1a6f63 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala @@ -40,17 +40,13 @@ class DataSourceV2DataFrameSuite import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ import testImplicits._ - override protected def sparkConf: SparkConf = - super.sparkConf.set(SQLConf.ANSI_ENABLED, true) - - before { - spark.conf.set("spark.sql.catalog.testcat", classOf[InMemoryTableCatalog].getName) - spark.conf.set("spark.sql.catalog.testcat2", classOf[InMemoryTableCatalog].getName) - } + override protected def sparkConf: SparkConf = super.sparkConf + .set(SQLConf.ANSI_ENABLED, true) + .set("spark.sql.catalog.testcat", classOf[InMemoryTableCatalog].getName) + .set("spark.sql.catalog.testcat2", classOf[InMemoryTableCatalog].getName) after { spark.sessionState.catalogManager.reset() - spark.sessionState.conf.clear() } override protected val catalogAndNamespace: String = "testcat.ns1.ns2.tbls" From 04a59edd8718a38e79450d1b2fd3a0ce0e08e015 Mon Sep 17 00:00:00 2001 From: Wenchen Fan Date: Thu, 5 Jun 2025 17:11:36 -0700 Subject: [PATCH 6/6] another fix --- .../apache/spark/sql/connector/DataSourceV2SQLSuite.scala | 5 ++++- .../org/apache/spark/sql/connector/DatasourceV2SQLBase.scala | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala index 48ccf33faaa1c..cd4cd462088a0 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala @@ -25,7 +25,7 @@ import java.util.Locale import scala.concurrent.duration.MICROSECONDS import scala.jdk.CollectionConverters._ -import org.apache.spark.{SparkException, SparkRuntimeException, SparkUnsupportedOperationException} +import org.apache.spark.{SparkConf, SparkException, SparkRuntimeException, SparkUnsupportedOperationException} import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.{InternalRow, QualifiedTableName, TableIdentifier} import org.apache.spark.sql.catalyst.CurrentUserContext.CURRENT_USER @@ -57,6 +57,9 @@ abstract class DataSourceV2SQLSuite with DeleteFromTests with DatasourceV2SQLBase with StatsEstimationTestBase with AdaptiveSparkPlanHelper { + override protected def sparkConf: SparkConf = + super.sparkConf.set(SQLConf.ANSI_ENABLED, true) + protected val v2Source = classOf[FakeV2ProviderWithCustomSchema].getName override protected val v2Format = v2Source diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DatasourceV2SQLBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DatasourceV2SQLBase.scala index 4ccff44fa0674..683d814d28b43 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DatasourceV2SQLBase.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DatasourceV2SQLBase.scala @@ -22,6 +22,7 @@ import org.scalatest.BeforeAndAfter import org.apache.spark.sql.QueryTest import org.apache.spark.sql.connector.catalog.{CatalogPlugin, InMemoryCatalog, InMemoryPartitionTableCatalog, InMemoryTableWithV2FilterCatalog, StagingInMemoryTableCatalog} import org.apache.spark.sql.connector.catalog.CatalogManager.SESSION_CATALOG_NAME +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession trait DatasourceV2SQLBase @@ -52,6 +53,7 @@ trait DatasourceV2SQLBase after { spark.sessionState.catalog.reset() spark.sessionState.catalogManager.reset() - spark.sessionState.conf.clear() + spark.sessionState.conf.unsetConf(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION) + spark.sessionState.conf.unsetConf(SQLConf.DEFAULT_CATALOG) } }