Skip to content

[SPARK-53319][SQL] Support the time type by try_make_timestamp_ltz() #52063

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 8 commits into
base: master
Choose a base branch
from
+206 −2
Open

[SPARK-53319][SQL] Support the time type by try_make_timestamp_ltz() #52063

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
9c6ee89
Initial commit
uros-db Aug 18, 2025
d91b621
Merge branch 'apache:master' into try_make_timestamp_ltz
uros-db Aug 25, 2025
d26a173
Regenerate golden files correctly
uros-db Aug 26, 2025
71f1a78
Add new expression examples
uros-db Aug 26, 2025
83a1931
Update expression arguments
uros-db Aug 27, 2025
cd135a9
Fix expression description
uros-db Aug 28, 2025
18dbffe
Use TryEval
uros-db Aug 28, 2025
543b563
Update sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expr…
cloud-fan Oct 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 22 additions & 2 deletions ...talyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2783,7 +2783,11 @@ object MakeTimestampLTZExpressionBuilder extends ExpressionBuilder {

// scalastyle:off line.size.limit
@ExpressionDescription(
usage = "_FUNC_(year, month, day, hour, min, sec[, timezone]) - Try to create the current timestamp with local time zone from year, month, day, hour, min, sec and timezone fields. The function returns NULL on invalid inputs.",
usage = """
_FUNC_(year, month, day, hour, min, sec[, timezone]) - Try to create the current timestamp with local time zone from year, month, day, hour, min, sec and (optional) timezone fields. The function returns NULL on invalid inputs.

_FUNC_(date, time[, timezone]) - Try to create the current timestamp with local time zone from date, time and (optional) timezone fields.
""",
arguments = """
Arguments:
* year - the year to represent, from 1 to 9999
Expand All @@ -2795,6 +2799,8 @@ object MakeTimestampLTZExpressionBuilder extends ExpressionBuilder {
0 to 60. If the sec argument equals to 60, the seconds field is set
to 0 and 1 minute is added to the final timestamp.
* timezone - the time zone identifier. For example, CET, UTC and etc.
* date - a date to represent, from 0001-01-01 to 9999-12-31
* time - a local time to represent, from 00:00:00 to 23:59:59.999999
""",
examples = """
Examples:
Expand All @@ -2808,14 +2814,28 @@ object MakeTimestampLTZExpressionBuilder extends ExpressionBuilder {
NULL
> SELECT _FUNC_(2024, 13, 22, 15, 30, 0);
NULL
> SELECT _FUNC_(DATE'2014-12-28', TIME'6:30:45.887');
2014-12-28 06:30:45.887
> SELECT _FUNC_(DATE'2014-12-28', TIME'6:30:45.887', 'CET');
2014-12-27 21:30:45.887
""",
group = "datetime_funcs",
since = "4.0.0")
// scalastyle:on line.size.limit
object TryMakeTimestampLTZExpressionBuilder extends ExpressionBuilder {
override def build(funcName: String, expressions: Seq[Expression]): Expression = {
val numArgs = expressions.length
if (numArgs == 6 || numArgs == 7) {
if (numArgs == 2 || numArgs == 3) {
// Overload for: date, time[, timezone].
TryEval(
MakeTimestampFromDateTime(
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

shall we add the failOnError flag to MakeTimestampFromDateTime as well? TryEval is not very reliable as it may mistakenly catch errors from the function argument expressions.

Copy link
Contributor Author

@uros-db uros-db Oct 14, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think we have to do that here, because invalid date/time here would throw parse exceptions before that point anyways (both in ANSI and non-ANSI). In this sense, it's a different kind of situation compared to integer construction (year, month, day, hour, min) of MakeTimestamp below for example.

expressions(0),
Some(expressions(1)),
expressions.drop(2).lastOption
)
)
}
else if (numArgs == 6 || numArgs == 7) {
MakeTimestamp(
expressions(0),
expressions(1),
Expand Down
81 changes: 81 additions & 0 deletions sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,3 +51,84 @@ Project [make_timestamp_ltz(2021, 7, 11, 6, 30, cast(60.007 as decimal(16,6)), N
SELECT convert_timezone('Europe/Brussels', timestamp_ltz'2022-03-23 00:00:00 America/Los_Angeles')
-- !query analysis
[Analyzer test output redacted due to nondeterminism]


-- !query
SELECT try_make_timestamp_ltz(make_date(2021, 07, 11), make_time(6, 30, 45.678))
-- !query analysis
Project [try_make_timestamp_ltz(make_timestamp(make_date(2021, 7, 11, true), Some(make_time(6, 30, cast(45.678 as decimal(16,6)))), None, Some(America/Los_Angeles))) AS try_make_timestamp_ltz(make_timestamp(make_date(2021, 7, 11), make_time(6, 30, 45.678)))#x]
+- OneRowRelation


-- !query
SELECT try_make_timestamp_ltz(NULL, TIME'00:00:00')
-- !query analysis
[Analyzer test output redacted due to nondeterminism]


-- !query
SELECT try_make_timestamp_ltz(DATE'1970-01-01', NULL)
-- !query analysis
[Analyzer test output redacted due to nondeterminism]


-- !query
SELECT try_make_timestamp_ltz(timestamp_ntz'2018-11-17 13:33:33', TIME'0:0:0')
-- !query analysis
org.apache.spark.sql.catalyst.ExtendedAnalysisException
{
"errorClass" : "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
"sqlState" : "42K09",
"messageParameters" : {
"inputSql" : "\"TIMESTAMP_NTZ '2018-11-17 13:33:33'\"",
"inputType" : "\"TIMESTAMP_NTZ\"",
"paramIndex" : "first",
"requiredType" : "\"DATE\"",
"sqlExpr" : "\"make_timestamp(TIMESTAMP_NTZ '2018-11-17 13:33:33', TIME '00:00:00')\""
},
"queryContext" : [ {
"objectType" : "",
"objectName" : "",
"startIndex" : 8,
"stopIndex" : 78,
"fragment" : "try_make_timestamp_ltz(timestamp_ntz'2018-11-17 13:33:33', TIME'0:0:0')"
} ]
}


-- !query
SELECT try_make_timestamp_ltz(DATE'2025-06-20', timestamp_ntz'2018-11-17 13:33:33')
-- !query analysis
org.apache.spark.sql.catalyst.ExtendedAnalysisException
{
"errorClass" : "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
"sqlState" : "42K09",
"messageParameters" : {
"inputSql" : "\"TIMESTAMP_NTZ '2018-11-17 13:33:33'\"",
"inputType" : "\"TIMESTAMP_NTZ\"",
"paramIndex" : "second",
"requiredType" : "\"TIME\"",
"sqlExpr" : "\"make_timestamp(DATE '2025-06-20', TIMESTAMP_NTZ '2018-11-17 13:33:33')\""
},
"queryContext" : [ {
"objectType" : "",
"objectName" : "",
"startIndex" : 8,
"stopIndex" : 83,
"fragment" : "try_make_timestamp_ltz(DATE'2025-06-20', timestamp_ntz'2018-11-17 13:33:33')"
} ]
}


-- !query
SELECT try_make_timestamp_ltz(make_date(2021, 07, 11), make_time(6, 30, 45.678), 'PST')
-- !query analysis
Project [try_make_timestamp_ltz(make_timestamp(make_date(2021, 7, 11, true), Some(make_time(6, 30, cast(45.678 as decimal(16,6)))), Some(PST), Some(America/Los_Angeles))) AS try_make_timestamp_ltz(make_timestamp(make_date(2021, 7, 11), make_time(6, 30, 45.678), PST))#x]
+- OneRowRelation


-- !query
SELECT try_make_timestamp_ltz(make_date(2021, 07, 11), make_time(6, 30, 45.678), 'CET')
-- !query analysis
Project [try_make_timestamp_ltz(make_timestamp(make_date(2021, 7, 11, true), Some(make_time(6, 30, cast(45.678 as decimal(16,6)))), Some(CET), Some(America/Los_Angeles))) AS try_make_timestamp_ltz(make_timestamp(make_date(2021, 7, 11), make_time(6, 30, 45.678), CET))#x]
+- OneRowRelation
12 changes: 12 additions & 0 deletions sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,3 +16,15 @@ SELECT make_timestamp_ltz(2021, 07, 11, 6, 30, 45.678, 'CET');
SELECT make_timestamp_ltz(2021, 07, 11, 6, 30, 60.007);

SELECT convert_timezone('Europe/Brussels', timestamp_ltz'2022-03-23 00:00:00 America/Los_Angeles');

-- Try TimestampLTZ date/time fields constructor
SELECT try_make_timestamp_ltz(make_date(2021, 07, 11), make_time(6, 30, 45.678));
-- Handling NULL input.
SELECT try_make_timestamp_ltz(NULL, TIME'00:00:00');
SELECT try_make_timestamp_ltz(DATE'1970-01-01', NULL);
-- Handling invalid input.
SELECT try_make_timestamp_ltz(timestamp_ntz'2018-11-17 13:33:33', TIME'0:0:0');
SELECT try_make_timestamp_ltz(DATE'2025-06-20', timestamp_ntz'2018-11-17 13:33:33');
-- Optional timezone is ignored.
SELECT try_make_timestamp_ltz(make_date(2021, 07, 11), make_time(6, 30, 45.678), 'PST');
SELECT try_make_timestamp_ltz(make_date(2021, 07, 11), make_time(6, 30, 45.678), 'CET');
92 changes: 92 additions & 0 deletions sql/core/src/test/resources/sql-tests/results/timestamp-ltz.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,3 +68,95 @@ SELECT convert_timezone('Europe/Brussels', timestamp_ltz'2022-03-23 00:00:00 Ame
struct<convert_timezone(current_timezone(), Europe/Brussels, TIMESTAMP '2022-03-23 00:00:00'):timestamp_ntz>
-- !query output
2022-03-23 08:00:00


-- !query
SELECT try_make_timestamp_ltz(make_date(2021, 07, 11), make_time(6, 30, 45.678))
-- !query schema
struct<try_make_timestamp_ltz(make_timestamp(make_date(2021, 7, 11), make_time(6, 30, 45.678))):timestamp>
-- !query output
2021-07-11 06:30:45.678


-- !query
SELECT try_make_timestamp_ltz(NULL, TIME'00:00:00')
-- !query schema
struct<try_make_timestamp_ltz(make_timestamp(NULL, TIME '00:00:00')):timestamp>
-- !query output
NULL


-- !query
SELECT try_make_timestamp_ltz(DATE'1970-01-01', NULL)
-- !query schema
struct<try_make_timestamp_ltz(make_timestamp(DATE '1970-01-01', NULL)):timestamp>
-- !query output
NULL


-- !query
SELECT try_make_timestamp_ltz(timestamp_ntz'2018-11-17 13:33:33', TIME'0:0:0')
-- !query schema
struct<>
-- !query output
org.apache.spark.sql.catalyst.ExtendedAnalysisException
{
"errorClass" : "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
"sqlState" : "42K09",
"messageParameters" : {
"inputSql" : "\"TIMESTAMP_NTZ '2018-11-17 13:33:33'\"",
"inputType" : "\"TIMESTAMP_NTZ\"",
"paramIndex" : "first",
"requiredType" : "\"DATE\"",
"sqlExpr" : "\"make_timestamp(TIMESTAMP_NTZ '2018-11-17 13:33:33', TIME '00:00:00')\""
},
"queryContext" : [ {
"objectType" : "",
"objectName" : "",
"startIndex" : 8,
"stopIndex" : 78,
"fragment" : "try_make_timestamp_ltz(timestamp_ntz'2018-11-17 13:33:33', TIME'0:0:0')"
} ]
}


-- !query
SELECT try_make_timestamp_ltz(DATE'2025-06-20', timestamp_ntz'2018-11-17 13:33:33')
-- !query schema
struct<>
-- !query output
org.apache.spark.sql.catalyst.ExtendedAnalysisException
{
"errorClass" : "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
"sqlState" : "42K09",
"messageParameters" : {
"inputSql" : "\"TIMESTAMP_NTZ '2018-11-17 13:33:33'\"",
"inputType" : "\"TIMESTAMP_NTZ\"",
"paramIndex" : "second",
"requiredType" : "\"TIME\"",
"sqlExpr" : "\"make_timestamp(DATE '2025-06-20', TIMESTAMP_NTZ '2018-11-17 13:33:33')\""
},
"queryContext" : [ {
"objectType" : "",
"objectName" : "",
"startIndex" : 8,
"stopIndex" : 83,
"fragment" : "try_make_timestamp_ltz(DATE'2025-06-20', timestamp_ntz'2018-11-17 13:33:33')"
} ]
}


-- !query
SELECT try_make_timestamp_ltz(make_date(2021, 07, 11), make_time(6, 30, 45.678), 'PST')
-- !query schema
struct<try_make_timestamp_ltz(make_timestamp(make_date(2021, 7, 11), make_time(6, 30, 45.678), PST)):timestamp>
-- !query output
2021-07-11 06:30:45.678


-- !query
SELECT try_make_timestamp_ltz(make_date(2021, 07, 11), make_time(6, 30, 45.678), 'CET')
-- !query schema
struct<try_make_timestamp_ltz(make_timestamp(make_date(2021, 7, 11), make_time(6, 30, 45.678), CET)):timestamp>
-- !query output
2021-07-10 21:30:45.678