diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveWithCTE.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveWithCTE.scala index 4a1a4215bebd9..b44c8c3ddcf1a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveWithCTE.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveWithCTE.scala @@ -20,11 +20,12 @@ package org.apache.spark.sql.catalyst.analysis import scala.collection.mutable import org.apache.spark.sql.AnalysisException -import org.apache.spark.sql.catalyst.expressions.SubqueryExpression +import org.apache.spark.sql.catalyst.expressions.{Alias, SubqueryExpression} import org.apache.spark.sql.catalyst.plans.{Cross, Inner, LeftAnti, LeftOuter, LeftSemi, RightOuter} import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.trees.TreePattern.{CTE, PLAN_EXPRESSION} +import org.apache.spark.sql.errors.QueryCompilationErrors /** * Updates CTE references with the resolve output attributes of corresponding CTE definitions. @@ -107,7 +108,7 @@ object ResolveWithCTE extends Rule[LogicalPlan] { // The case of CTE name followed by a parenthesized list of column name(s), eg. // WITH RECURSIVE t(n). case alias @ SubqueryAlias(_, - columnAlias @ UnresolvedSubqueryColumnAliases( + _ @ UnresolvedSubqueryColumnAliases( colNames, Union(Seq(anchor, recursion), false, false) )) => @@ -116,16 +117,16 @@ object ResolveWithCTE extends Rule[LogicalPlan] { } else { val loop = UnionLoop( cteDef.id, - anchor, + UnresolvedSubqueryColumnAliases(colNames, anchor), rewriteRecursiveCTERefs(recursion, anchor, cteDef.id, Some(colNames)), maxDepth = cteDef.maxDepth) - cteDef.copy(child = alias.copy(child = columnAlias.copy(child = loop))) + cteDef.copy(child = alias.copy(child = loop)) } // The case of CTE name followed by a parenthesized list of column name(s), eg. // WITH RECURSIVE t(n). case alias @ SubqueryAlias(_, - columnAlias @ UnresolvedSubqueryColumnAliases( + _ @ UnresolvedSubqueryColumnAliases( colNames, withCTE @ WithCTE(Union(Seq(anchor, recursion), false, false), innerCteDefs) )) => @@ -140,11 +141,11 @@ object ResolveWithCTE extends Rule[LogicalPlan] { } val loop = UnionLoop( cteDef.id, - anchor, + UnresolvedSubqueryColumnAliases(colNames, anchor), rewriteRecursiveCTERefs(recursion, anchor, cteDef.id, Some(colNames)), maxDepth = cteDef.maxDepth) - cteDef.copy(child = alias.copy(child = columnAlias.copy( - child = withCTE.copy(plan = loop, cteDefs = newInnerCteDefs)))) + cteDef.copy(child = alias.copy(child = + withCTE.copy(plan = loop, cteDefs = newInnerCteDefs))) } // If the recursion is described with a UNION (deduplicating) clause then the @@ -202,7 +203,7 @@ object ResolveWithCTE extends Rule[LogicalPlan] { // The case of CTE name followed by a parenthesized list of column name(s). case alias @ SubqueryAlias(_, - columnAlias@UnresolvedSubqueryColumnAliases( + _ @ UnresolvedSubqueryColumnAliases( colNames, Distinct(Union(Seq(anchor, recursion), false, false)) )) => @@ -214,7 +215,7 @@ object ResolveWithCTE extends Rule[LogicalPlan] { } else { val loop = UnionLoop( cteDef.id, - Distinct(anchor), + UnresolvedSubqueryColumnAliases(colNames, Distinct(anchor)), Except( rewriteRecursiveCTERefs(recursion, anchor, cteDef.id, Some(colNames)), UnionLoopRef(cteDef.id, anchor.output, true), @@ -222,12 +223,12 @@ object ResolveWithCTE extends Rule[LogicalPlan] { ), maxDepth = cteDef.maxDepth ) - cteDef.copy(child = alias.copy(child = columnAlias.copy(child = loop))) + cteDef.copy(child = alias.copy(child = loop)) } // The case of CTE name followed by a parenthesized list of column name(s). case alias @ SubqueryAlias(_, - columnAlias@UnresolvedSubqueryColumnAliases( + _ @ UnresolvedSubqueryColumnAliases( colNames, WithCTE(Distinct(Union(Seq(anchor, recursion), false, false)), innerCteDefs) )) => @@ -245,7 +246,7 @@ object ResolveWithCTE extends Rule[LogicalPlan] { } val loop = UnionLoop( cteDef.id, - Distinct(anchor), + UnresolvedSubqueryColumnAliases(colNames, Distinct(anchor)), Except( rewriteRecursiveCTERefs(recursion, anchor, cteDef.id, Some(colNames)), UnionLoopRef(cteDef.id, anchor.output, true), @@ -253,8 +254,8 @@ object ResolveWithCTE extends Rule[LogicalPlan] { ), maxDepth = cteDef.maxDepth ) - cteDef.copy(child = alias.copy(child = columnAlias.copy( - child = withCTE.copy(plan = loop, cteDefs = newInnerCteDefs)))) + cteDef.copy(child = alias.copy(child = + withCTE.copy(plan = loop, cteDefs = newInnerCteDefs))) } case other => @@ -298,8 +299,24 @@ object ResolveWithCTE extends Rule[LogicalPlan] { columnNames: Option[Seq[String]]) = { recursion.transformUpWithSubqueriesAndPruning(_.containsPattern(CTE)) { case r: CTERelationRef if r.recursive && r.cteId == cteDefId => - val ref = UnionLoopRef(r.cteId, anchor.output, false) - columnNames.map(UnresolvedSubqueryColumnAliases(_, ref)).getOrElse(ref) + columnNames match { + case Some(names) => + val outputAttrs = anchor.output + // Checks if the number of the aliases equals to the number of output columns + // in the subquery. + if (names.size != outputAttrs.size) { + throw QueryCompilationErrors.aliasNumberNotMatchColumnNumberError( + names.size, outputAttrs.size, r) + } + val aliases = outputAttrs.zip(names).map { case (attr, aliasName) => + Alias(attr, aliasName)() + } + UnionLoopRef(r.cteId, aliases.map(_.toAttribute), accumulated = false) + + case None => + UnionLoopRef(r.cteId, anchor.output, accumulated = false) + + } } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveRecursiveCTESuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveRecursiveCTESuite.scala index e76e261223ebc..c5838c1df01b5 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveRecursiveCTESuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveRecursiveCTESuite.scala @@ -77,11 +77,11 @@ class ResolveRecursiveCTESuite extends AnalysisTest { def getAfterPlan(): LogicalPlan = { val col = anchor.output.head - val recursion = UnionLoopRef(cteId, anchor.output, accumulated = false) - .select(col.as("n")) + val newAnchor = anchor.select(col.as("n")) + val recursion = UnionLoopRef(cteId, newAnchor.output, accumulated = false) .subquery("t") val cteDef = CTERelationDef( - UnionLoop(cteId, anchor, recursion).select(col.as("n")).subquery("t"), + UnionLoop(cteId, newAnchor, recursion).subquery("t"), cteId) val cteRef = CTERelationRef( cteId, diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/cte-recursion.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/cte-recursion.sql.out index 4aff038838654..22fe26f6b58a2 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/cte-recursion.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/cte-recursion.sql.out @@ -58,14 +58,13 @@ SELECT * FROM r WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias r -: +- Project [col1#x AS level#x] -: +- UnionLoop xxxx -: :- LocalRelation [col1#x] -: +- Project [(level#x + 1) AS (level + 1)#x] -: +- Filter (level#x < 9) -: +- SubqueryAlias r -: +- Project [col1#x AS level#x] -: +- UnionLoopRef xxxx, [col1#x], false +: +- UnionLoop xxxx +: :- Project [col1#x AS level#x] +: : +- LocalRelation [col1#x] +: +- Project [(level#x + 1) AS (level + 1)#x] +: +- Filter (level#x < 9) +: +- SubqueryAlias r +: +- UnionLoopRef xxxx, [level#x], false +- Project [level#x] +- SubqueryAlias r +- CTERelationRef xxxx, true, [level#x], false, false @@ -82,15 +81,14 @@ SELECT * FROM r WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias r -: +- Project [a#x AS c#x] -: +- UnionLoop xxxx -: :- Project [a AS a#x] -: : +- OneRowRelation -: +- Project [concat(c#x, char(cast((ascii(substr(c#x, -1, 2147483647)) + 1) as bigint))) AS concat(c, char((ascii(substr(c, -1, 2147483647)) + 1)))#x] -: +- Filter (length(c#x) < 10) -: +- SubqueryAlias r -: +- Project [a#x AS c#x] -: +- UnionLoopRef xxxx, [a#x], false +: +- UnionLoop xxxx +: :- Project [a#x AS c#x] +: : +- Project [a AS a#x] +: : +- OneRowRelation +: +- Project [concat(c#x, char(cast((ascii(substr(c#x, -1, 2147483647)) + 1) as bigint))) AS concat(c, char((ascii(substr(c, -1, 2147483647)) + 1)))#x] +: +- Filter (length(c#x) < 10) +: +- SubqueryAlias r +: +- UnionLoopRef xxxx, [c#x], false +- Project [c#x] +- SubqueryAlias r +- CTERelationRef xxxx, true, [c#x], false, false @@ -107,13 +105,12 @@ SELECT * FROM r WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias r -: +- Project [col1#x AS level#x] -: +- UnionLoop xxxx -: :- LocalRelation [col1#x] -: +- Project [(level#x + 1) AS (level + 1)#x] -: +- SubqueryAlias r -: +- Project [col1#x AS level#x] -: +- UnionLoopRef xxxx, [col1#x], false +: +- UnionLoop xxxx +: :- Project [col1#x AS level#x] +: : +- LocalRelation [col1#x] +: +- Project [(level#x + 1) AS (level + 1)#x] +: +- SubqueryAlias r +: +- UnionLoopRef xxxx, [level#x], false +- Project [level#x] +- SubqueryAlias r +- CTERelationRef xxxx, true, [level#x], false, false @@ -145,19 +142,18 @@ SELECT * FROM t WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias t -: +- Project [1#x AS n#x] -: +- UnionLoop xxxx -: :- Project [1 AS 1#x] -: : +- OneRowRelation -: +- Project [next#x] -: +- Join LeftOuter, (n#x = current#x) -: :- SubqueryAlias t -: : +- Project [1#x AS n#x] -: : +- UnionLoopRef xxxx, [1#x], false -: +- SubqueryAlias zeroandone -: +- View (`ZeroAndOne`, [current#x, next#x]) -: +- Project [cast(col1#x as int) AS current#x, cast(col2#x as int) AS next#x] -: +- LocalRelation [col1#x, col2#x] +: +- UnionLoop xxxx +: :- Project [1#x AS n#x] +: : +- Project [1 AS 1#x] +: : +- OneRowRelation +: +- Project [next#x] +: +- Join LeftOuter, (n#x = current#x) +: :- SubqueryAlias t +: : +- UnionLoopRef xxxx, [n#x], false +: +- SubqueryAlias zeroandone +: +- View (`ZeroAndOne`, [current#x, next#x]) +: +- Project [cast(col1#x as int) AS current#x, cast(col2#x as int) AS next#x] +: +- LocalRelation [col1#x, col2#x] +- Project [n#x] +- SubqueryAlias t +- CTERelationRef xxxx, true, [n#x], false, false @@ -180,14 +176,13 @@ SELECT * FROM r WithCTE :- CTERelationDef xxxx, false, 35 : +- SubqueryAlias r -: +- Project [col1#x AS level#x] -: +- UnionLoop xxxx, 35 -: :- LocalRelation [col1#x] -: +- Project [(level#x + 1) AS (level + 1)#x] -: +- Filter (level#x < 30) -: +- SubqueryAlias r -: +- Project [col1#x AS level#x] -: +- UnionLoopRef xxxx, [col1#x], false +: +- UnionLoop xxxx, 35 +: :- Project [col1#x AS level#x] +: : +- LocalRelation [col1#x] +: +- Project [(level#x + 1) AS (level + 1)#x] +: +- Filter (level#x < 30) +: +- SubqueryAlias r +: +- UnionLoopRef xxxx, [level#x], false +- Project [level#x] +- SubqueryAlias r +- CTERelationRef xxxx, true, [level#x], false, false @@ -228,14 +223,13 @@ SELECT * FROM r WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias r -: +- Project [col1#x AS level#x] -: +- UnionLoop xxxx -: :- LocalRelation [col1#x] -: +- Project [(level#x + 1) AS (level + 1)#x] -: +- Filter (level#x < 150) -: +- SubqueryAlias r -: +- Project [col1#x AS level#x] -: +- UnionLoopRef xxxx, [col1#x], false +: +- UnionLoop xxxx +: :- Project [col1#x AS level#x] +: : +- LocalRelation [col1#x] +: +- Project [(level#x + 1) AS (level + 1)#x] +: +- Filter (level#x < 150) +: +- SubqueryAlias r +: +- UnionLoopRef xxxx, [level#x], false +- Project [level#x] +- SubqueryAlias r +- CTERelationRef xxxx, true, [level#x], false, false @@ -267,19 +261,18 @@ SELECT * FROM t LIMIT 60 WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias t -: +- Project [1#x AS n#x] -: +- UnionLoop xxxx -: :- Project [1 AS 1#x] -: : +- OneRowRelation -: +- Project [next#x] -: +- Join LeftOuter, (n#x = current#x) -: :- SubqueryAlias t -: : +- Project [1#x AS n#x] -: : +- UnionLoopRef xxxx, [1#x], false -: +- SubqueryAlias zeroandone -: +- View (`ZeroAndOne`, [current#x, next#x]) -: +- Project [cast(col1#x as int) AS current#x, cast(col2#x as int) AS next#x] -: +- LocalRelation [col1#x, col2#x] +: +- UnionLoop xxxx +: :- Project [1#x AS n#x] +: : +- Project [1 AS 1#x] +: : +- OneRowRelation +: +- Project [next#x] +: +- Join LeftOuter, (n#x = current#x) +: :- SubqueryAlias t +: : +- UnionLoopRef xxxx, [n#x], false +: +- SubqueryAlias zeroandone +: +- View (`ZeroAndOne`, [current#x, next#x]) +: +- Project [cast(col1#x as int) AS current#x, cast(col2#x as int) AS next#x] +: +- LocalRelation [col1#x, col2#x] +- GlobalLimit 60 +- LocalLimit 60 +- Project [n#x] @@ -304,13 +297,12 @@ SELECT * FROM r LIMIT 10 WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias r -: +- Project [col1#x AS level#x] -: +- UnionLoop xxxx -: :- LocalRelation [col1#x] -: +- Project [(level#x + 1) AS (level + 1)#x] -: +- SubqueryAlias r -: +- Project [col1#x AS level#x] -: +- UnionLoopRef xxxx, [col1#x], false +: +- UnionLoop xxxx +: :- Project [col1#x AS level#x] +: : +- LocalRelation [col1#x] +: +- Project [(level#x + 1) AS (level + 1)#x] +: +- SubqueryAlias r +: +- UnionLoopRef xxxx, [level#x], false +- GlobalLimit 10 +- LocalLimit 10 +- Project [level#x] @@ -373,13 +365,12 @@ SELECT level, level FROM r ORDER BY 1 LIMIT 10 WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias r -: +- Project [col1#x AS level#x] -: +- UnionLoop xxxx -: :- LocalRelation [col1#x] -: +- Project [(level#x + 1) AS (level + 1)#x] -: +- SubqueryAlias r -: +- Project [col1#x AS level#x] -: +- UnionLoopRef xxxx, [col1#x], false +: +- UnionLoop xxxx +: :- Project [col1#x AS level#x] +: : +- LocalRelation [col1#x] +: +- Project [(level#x + 1) AS (level + 1)#x] +: +- SubqueryAlias r +: +- UnionLoopRef xxxx, [level#x], false +- GlobalLimit 10 +- LocalLimit 10 +- Sort [level#x ASC NULLS FIRST], true @@ -474,16 +465,15 @@ SELECT * FROM t LIMIT 5 WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias t -: +- Project [1#x AS col#x] -: +- UnionLoop xxxx -: :- Project [1 AS 1#x] -: : +- OneRowRelation -: +- Project [scalar-subquery#x [] AS scalarsubquery()#x] -: : +- Aggregate [max(col#x) AS max(col)#x] -: : +- SubqueryAlias t -: : +- Project [1#x AS col#x] -: : +- UnionLoopRef xxxx, [1#x], false -: +- OneRowRelation +: +- UnionLoop xxxx +: :- Project [1#x AS col#x] +: : +- Project [1 AS 1#x] +: : +- OneRowRelation +: +- Project [scalar-subquery#x [] AS scalarsubquery()#x] +: : +- Aggregate [max(col#x) AS max(col)#x] +: : +- SubqueryAlias t +: : +- UnionLoopRef xxxx, [col#x], false +: +- OneRowRelation +- GlobalLimit 5 +- LocalLimit 5 +- Project [col#x] @@ -615,20 +605,18 @@ SELECT * FROM r WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias r -: +- Project [col1#x AS level#x, col2#x AS data#x] -: +- UnionLoop xxxx -: :- LocalRelation [col1#x, col2#x] -: +- Project [(level#x + 1) AS (level + 1)#x, data#x] -: +- Filter (level#x < 9) -: +- Join Inner, (data#x = data#x) -: :- SubqueryAlias r1 -: : +- SubqueryAlias r -: : +- Project [col1#x AS level#x, col2#x AS data#x] -: : +- UnionLoopRef xxxx, [col1#x, col2#x], false -: +- SubqueryAlias r2 -: +- SubqueryAlias r -: +- Project [col1#x AS level#x, col2#x AS data#x] -: +- UnionLoopRef xxxx, [col1#x, col2#x], false +: +- UnionLoop xxxx +: :- Project [col1#x AS level#x, col2#x AS data#x] +: : +- LocalRelation [col1#x, col2#x] +: +- Project [(level#x + 1) AS (level + 1)#x, data#x] +: +- Filter (level#x < 9) +: +- Join Inner, (data#x = data#x) +: :- SubqueryAlias r1 +: : +- SubqueryAlias r +: : +- UnionLoopRef xxxx, [level#x, data#x], false +: +- SubqueryAlias r2 +: +- SubqueryAlias r +: +- UnionLoopRef xxxx, [level#x, data#x], false +- Project [level#x, data#x] +- SubqueryAlias r +- CTERelationRef xxxx, true, [level#x, data#x], false, false @@ -650,18 +638,17 @@ SELECT * FROM r WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias r -: +- Project [col1#x AS level#x, col2#x AS data#x] -: +- UnionLoop xxxx -: :- LocalRelation [col1#x, col2#x] -: +- Project [(level#x + 1) AS (level + 1)#x, data#x] -: +- Filter (level#x < 9) -: +- Join LeftOuter, (data#x = data#x) -: :- SubqueryAlias r -: : +- Project [col1#x AS level#x, col2#x AS data#x] -: : +- UnionLoopRef xxxx, [col1#x, col2#x], false -: +- SubqueryAlias t -: +- Project [0 AS data#x] -: +- OneRowRelation +: +- UnionLoop xxxx +: :- Project [col1#x AS level#x, col2#x AS data#x] +: : +- LocalRelation [col1#x, col2#x] +: +- Project [(level#x + 1) AS (level + 1)#x, data#x] +: +- Filter (level#x < 9) +: +- Join LeftOuter, (data#x = data#x) +: :- SubqueryAlias r +: : +- UnionLoopRef xxxx, [level#x, data#x], false +: +- SubqueryAlias t +: +- Project [0 AS data#x] +: +- OneRowRelation +- Project [level#x, data#x] +- SubqueryAlias r +- CTERelationRef xxxx, true, [level#x, data#x], false, false @@ -703,18 +690,17 @@ SELECT * FROM r WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias r -: +- Project [col1#x AS level#x, col2#x AS data#x] -: +- UnionLoop xxxx -: :- LocalRelation [col1#x, col2#x] -: +- Project [(level#x + 1) AS (level + 1)#x, data#x] -: +- Filter (level#x < 9) -: +- Join RightOuter, (data#x = data#x) -: :- SubqueryAlias t -: : +- Project [0 AS data#x] -: : +- OneRowRelation -: +- SubqueryAlias r -: +- Project [col1#x AS level#x, col2#x AS data#x] -: +- UnionLoopRef xxxx, [col1#x, col2#x], false +: +- UnionLoop xxxx +: :- Project [col1#x AS level#x, col2#x AS data#x] +: : +- LocalRelation [col1#x, col2#x] +: +- Project [(level#x + 1) AS (level + 1)#x, data#x] +: +- Filter (level#x < 9) +: +- Join RightOuter, (data#x = data#x) +: :- SubqueryAlias t +: : +- Project [0 AS data#x] +: : +- OneRowRelation +: +- SubqueryAlias r +: +- UnionLoopRef xxxx, [level#x, data#x], false +- Project [level#x, data#x] +- SubqueryAlias r +- CTERelationRef xxxx, true, [level#x, data#x], false, false @@ -776,18 +762,17 @@ SELECT * FROM r WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias r -: +- Project [col1#x AS level#x, col2#x AS data#x] -: +- UnionLoop xxxx -: :- LocalRelation [col1#x, col2#x] -: +- Project [(level#x + 1) AS (level + 1)#x, data#x] -: +- Filter (level#x < 9) -: +- Join LeftSemi, (data#x = data#x) -: :- SubqueryAlias r -: : +- Project [col1#x AS level#x, col2#x AS data#x] -: : +- UnionLoopRef xxxx, [col1#x, col2#x], false -: +- SubqueryAlias t -: +- Project [0 AS data#x] -: +- OneRowRelation +: +- UnionLoop xxxx +: :- Project [col1#x AS level#x, col2#x AS data#x] +: : +- LocalRelation [col1#x, col2#x] +: +- Project [(level#x + 1) AS (level + 1)#x, data#x] +: +- Filter (level#x < 9) +: +- Join LeftSemi, (data#x = data#x) +: :- SubqueryAlias r +: : +- UnionLoopRef xxxx, [level#x, data#x], false +: +- SubqueryAlias t +: +- Project [0 AS data#x] +: +- OneRowRelation +- Project [level#x, data#x] +- SubqueryAlias r +- CTERelationRef xxxx, true, [level#x, data#x], false, false @@ -828,18 +813,17 @@ SELECT * FROM r WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias r -: +- Project [col1#x AS level#x, col2#x AS data#x] -: +- UnionLoop xxxx -: :- LocalRelation [col1#x, col2#x] -: +- Project [(level#x + 1) AS (level + 1)#x, data#x] -: +- Filter (level#x < 9) -: +- Join LeftAnti, (data#x = data#x) -: :- SubqueryAlias r -: : +- Project [col1#x AS level#x, col2#x AS data#x] -: : +- UnionLoopRef xxxx, [col1#x, col2#x], false -: +- SubqueryAlias t -: +- Project [-1 AS data#x] -: +- OneRowRelation +: +- UnionLoop xxxx +: :- Project [col1#x AS level#x, col2#x AS data#x] +: : +- LocalRelation [col1#x, col2#x] +: +- Project [(level#x + 1) AS (level + 1)#x, data#x] +: +- Filter (level#x < 9) +: +- Join LeftAnti, (data#x = data#x) +: :- SubqueryAlias r +: : +- UnionLoopRef xxxx, [level#x, data#x], false +: +- SubqueryAlias t +: +- Project [-1 AS data#x] +: +- OneRowRelation +- Project [level#x, data#x] +- SubqueryAlias r +- CTERelationRef xxxx, true, [level#x, data#x], false, false @@ -911,14 +895,13 @@ SELECT * FROM r, false, false, LocalTempView, UNSUPPORTED, true +- WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias r - : +- Project [col1#x AS level#x] - : +- UnionLoop xxxx - : :- LocalRelation [col1#x] - : +- Project [(level#x + 1) AS (level + 1)#x] - : +- Filter (level#x < 9) - : +- SubqueryAlias r - : +- Project [col1#x AS level#x] - : +- UnionLoopRef xxxx, [col1#x], false + : +- UnionLoop xxxx + : :- Project [col1#x AS level#x] + : : +- LocalRelation [col1#x] + : +- Project [(level#x + 1) AS (level + 1)#x] + : +- Filter (level#x < 9) + : +- SubqueryAlias r + : +- UnionLoopRef xxxx, [level#x], false +- Project [level#x] +- SubqueryAlias r +- CTERelationRef xxxx, true, [level#x], false, false @@ -934,14 +917,13 @@ Project [level#x] +- WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias r - : +- Project [col1#x AS level#x] - : +- UnionLoop xxxx - : :- LocalRelation [col1#x] - : +- Project [(level#x + 1) AS (level + 1)#x] - : +- Filter (level#x < 9) - : +- SubqueryAlias r - : +- Project [col1#x AS level#x] - : +- UnionLoopRef xxxx, [col1#x], false + : +- UnionLoop xxxx + : :- Project [col1#x AS level#x] + : : +- LocalRelation [col1#x] + : +- Project [(level#x + 1) AS (level + 1)#x] + : +- Filter (level#x < 9) + : +- SubqueryAlias r + : +- UnionLoopRef xxxx, [level#x], false +- Project [level#x] +- SubqueryAlias r +- CTERelationRef xxxx, true, [level#x], false, false @@ -971,14 +953,13 @@ InsertIntoHadoopFsRelationCommand file:[not included in comparison]/{warehouse_d +- WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias r - : +- Project [col1#x AS level#x] - : +- UnionLoop xxxx - : :- LocalRelation [col1#x] - : +- Project [(level#x + 1) AS (level + 1)#x] - : +- Filter (level#x < 9) - : +- SubqueryAlias r - : +- Project [col1#x AS level#x] - : +- UnionLoopRef xxxx, [col1#x], false + : +- UnionLoop xxxx + : :- Project [col1#x AS level#x] + : : +- LocalRelation [col1#x] + : +- Project [(level#x + 1) AS (level + 1)#x] + : +- Filter (level#x < 9) + : +- SubqueryAlias r + : +- UnionLoopRef xxxx, [level#x], false +- Project [level#x] +- SubqueryAlias r +- CTERelationRef xxxx, true, [level#x], false, false @@ -1018,14 +999,13 @@ INSERT INTO rt2 SELECT * WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias r -: +- Project [col1#x AS level#x] -: +- UnionLoop xxxx -: :- LocalRelation [col1#x] -: +- Project [(level#x + 1) AS (level + 1)#x] -: +- Filter (level#x < 9) -: +- SubqueryAlias r -: +- Project [col1#x AS level#x] -: +- UnionLoopRef xxxx, [col1#x], false +: +- UnionLoop xxxx +: :- Project [col1#x AS level#x] +: : +- LocalRelation [col1#x] +: +- Project [(level#x + 1) AS (level + 1)#x] +: +- Filter (level#x < 9) +: +- SubqueryAlias r +: +- UnionLoopRef xxxx, [level#x], false +- Union false, false :- InsertIntoHadoopFsRelationCommand file:[not included in comparison]/{warehouse_dir}/rt2, false, CSV, [path=file:[not included in comparison]/{warehouse_dir}/rt2], Append, `spark_catalog`.`default`.`rt2`, org.apache.spark.sql.execution.datasources.InMemoryFileIndex(file:[not included in comparison]/{warehouse_dir}/rt2), [level] : +- Project [level#x] @@ -1184,14 +1164,13 @@ WithCTE : +- WithCTE : :- CTERelationDef xxxx, false : : +- SubqueryAlias r -: : +- Project [col1#x AS level#x] -: : +- UnionLoop xxxx -: : :- LocalRelation [col1#x] -: : +- Project [(level#x + 1) AS (level + 1)#x] -: : +- Filter (level#x < 3) -: : +- SubqueryAlias r -: : +- Project [col1#x AS level#x] -: : +- UnionLoopRef xxxx, [col1#x], false +: : +- UnionLoop xxxx +: : :- Project [col1#x AS level#x] +: : : +- LocalRelation [col1#x] +: : +- Project [(level#x + 1) AS (level + 1)#x] +: : +- Filter (level#x < 3) +: : +- SubqueryAlias r +: : +- UnionLoopRef xxxx, [level#x], false : +- Union false, false : :- Project [level#x] : : +- SubqueryAlias r @@ -1221,27 +1200,25 @@ SELECT * FROM r WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias r -: +- Project [level#x AS level#x] -: +- UnionLoop xxxx -: :- WithCTE -: : :- CTERelationDef xxxx, false -: : : +- SubqueryAlias r -: : : +- Project [col1#x AS level#x] -: : : +- UnionLoop xxxx -: : : :- LocalRelation [col1#x] -: : : +- Project [(level#x + 1) AS (level + 1)#x] -: : : +- Filter (level#x < 3) -: : : +- SubqueryAlias r -: : : +- Project [col1#x AS level#x] -: : : +- UnionLoopRef xxxx, [col1#x], false -: : +- Project [level#x] -: : +- SubqueryAlias r -: : +- CTERelationRef xxxx, true, [level#x], false, false -: +- Project [(level#x + 1) AS (level + 1)#x] -: +- Filter (level#x < 3) -: +- SubqueryAlias r -: +- Project [level#x AS level#x] -: +- UnionLoopRef xxxx, [level#x], false +: +- UnionLoop xxxx +: :- Project [level#x AS level#x] +: : +- WithCTE +: : :- CTERelationDef xxxx, false +: : : +- SubqueryAlias r +: : : +- UnionLoop xxxx +: : : :- Project [col1#x AS level#x] +: : : : +- LocalRelation [col1#x] +: : : +- Project [(level#x + 1) AS (level + 1)#x] +: : : +- Filter (level#x < 3) +: : : +- SubqueryAlias r +: : : +- UnionLoopRef xxxx, [level#x], false +: : +- Project [level#x] +: : +- SubqueryAlias r +: : +- CTERelationRef xxxx, true, [level#x], false, false +: +- Project [(level#x + 1) AS (level + 1)#x] +: +- Filter (level#x < 3) +: +- SubqueryAlias r +: +- UnionLoopRef xxxx, [level#x], false +- Project [level#x] +- SubqueryAlias r +- CTERelationRef xxxx, true, [level#x], false, false @@ -1335,14 +1312,13 @@ SELECT a FROM t1 LIMIT 5 WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias t1 -: +- Project [1#x AS a#x, 1#x AS b#x, 1#x AS c#x] -: +- UnionLoop xxxx -: :- Project [1 AS 1#x, 1 AS 1#x, 1 AS 1#x] -: : +- OneRowRelation -: +- Project [(a#x + 1) AS (a + 1)#x, (a#x + 1) AS (a + 1)#x, (a#x + 1) AS (a + 1)#x] -: +- SubqueryAlias t1 -: +- Project [1#x AS a#x, 1#x AS b#x, 1#x AS c#x] -: +- UnionLoopRef xxxx, [1#x, 1#x, 1#x], false +: +- UnionLoop xxxx +: :- Project [1#x AS a#x, 1#x AS b#x, 1#x AS c#x] +: : +- Project [1 AS 1#x, 1 AS 1#x, 1 AS 1#x] +: : +- OneRowRelation +: +- Project [(a#x + 1) AS (a + 1)#x, (a#x + 1) AS (a + 1)#x, (a#x + 1) AS (a + 1)#x] +: +- SubqueryAlias t1 +: +- UnionLoopRef xxxx, [a#x, b#x, c#x], false +- GlobalLimit 5 +- LocalLimit 5 +- Project [a#x] @@ -1375,17 +1351,16 @@ SELECT * FROM t LIMIT 63 WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias t -: +- Project [1#x AS n#x] -: +- UnionLoop xxxx -: :- Project [1 AS 1#x] -: : +- OneRowRelation -: +- Project [next#x] -: +- Join Cross -: :- SubqueryAlias t -: : +- Project [1#x AS n#x] -: : +- UnionLoopRef xxxx, [1#x], false -: +- SubqueryAlias spark_catalog.default.tb -: +- Relation spark_catalog.default.tb[next#x] json +: +- UnionLoop xxxx +: :- Project [1#x AS n#x] +: : +- Project [1 AS 1#x] +: : +- OneRowRelation +: +- Project [next#x] +: +- Join Cross +: :- SubqueryAlias t +: : +- UnionLoopRef xxxx, [n#x], false +: +- SubqueryAlias spark_catalog.default.tb +: +- Relation spark_catalog.default.tb[next#x] json +- GlobalLimit 63 +- LocalLimit 63 +- Project [n#x] @@ -1422,18 +1397,17 @@ WithCTE : +- OneRowRelation :- CTERelationDef xxxx, false : +- SubqueryAlias t -: +- Project [id#x AS id#x, xid#x AS xid#x] -: +- UnionLoop xxxx -: :- Project [0 AS id#x, 0 AS xid#x] -: : +- OneRowRelation -: +- Project [(id#x + 1) AS (id + 1)#x, ((xid#x * 10) + id#x) AS ((xid * 10) + id)#x] -: +- Filter (id#x < 3) -: +- Join Cross -: :- SubqueryAlias t -: : +- Project [id#x AS id#x, xid#x AS xid#x] -: : +- UnionLoopRef xxxx, [id#x, xid#x], false -: +- SubqueryAlias x -: +- CTERelationRef xxxx, true, [id#x], false, false, 2 +: +- UnionLoop xxxx +: :- Project [id#x AS id#x, xid#x AS xid#x] +: : +- Project [0 AS id#x, 0 AS xid#x] +: : +- OneRowRelation +: +- Project [(id#x + 1) AS (id + 1)#x, ((xid#x * 10) + id#x) AS ((xid * 10) + id)#x] +: +- Filter (id#x < 3) +: +- Join Cross +: :- SubqueryAlias t +: : +- UnionLoopRef xxxx, [id#x, xid#x], false +: +- SubqueryAlias x +: +- CTERelationRef xxxx, true, [id#x], false, false, 2 +- Project [id#x, xid#x] +- SubqueryAlias t +- CTERelationRef xxxx, true, [id#x, xid#x], false, false @@ -1455,29 +1429,27 @@ SELECT * FROM t2 WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias t1 -: +- Project [1#x AS a#x, 1#x AS b#x] -: +- UnionLoop xxxx -: :- Project [1 AS 1#x, 1 AS 1#x] -: : +- OneRowRelation -: +- Project [(a#x + b#x) AS (a + b)#x, a#x] -: +- Filter (a#x < 20) -: +- SubqueryAlias t1 -: +- Project [1#x AS a#x, 1#x AS b#x] -: +- UnionLoopRef xxxx, [1#x, 1#x], false +: +- UnionLoop xxxx +: :- Project [1#x AS a#x, 1#x AS b#x] +: : +- Project [1 AS 1#x, 1 AS 1#x] +: : +- OneRowRelation +: +- Project [(a#x + b#x) AS (a + b)#x, a#x] +: +- Filter (a#x < 20) +: +- SubqueryAlias t1 +: +- UnionLoopRef xxxx, [a#x, b#x], false :- CTERelationDef xxxx, false : +- SubqueryAlias t2 -: +- Project [1#x AS n#x] -: +- UnionLoop xxxx -: :- Project [1 AS 1#x] -: : +- OneRowRelation -: +- Project [(n#x + 1) AS (n + 1)#x] -: +- Filter ((n#x + 1) = a#x) -: +- Join Inner -: :- SubqueryAlias t2 -: : +- Project [1#x AS n#x] -: : +- UnionLoopRef xxxx, [1#x], false -: +- SubqueryAlias t1 -: +- CTERelationRef xxxx, true, [a#x, b#x], false, false +: +- UnionLoop xxxx +: :- Project [1#x AS n#x] +: : +- Project [1 AS 1#x] +: : +- OneRowRelation +: +- Project [(n#x + 1) AS (n + 1)#x] +: +- Filter ((n#x + 1) = a#x) +: +- Join Inner +: :- SubqueryAlias t2 +: : +- UnionLoopRef xxxx, [n#x], false +: +- SubqueryAlias t1 +: +- CTERelationRef xxxx, true, [a#x, b#x], false, false +- Project [n#x] +- SubqueryAlias t2 +- CTERelationRef xxxx, true, [n#x], false, false @@ -1502,27 +1474,26 @@ SELECT n FROM t1 WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias t1 -: +- Project [col1#x AS n#x] -: +- UnionLoop xxxx -: :- LocalRelation [col1#x] -: +- WithCTE -: :- CTERelationDef xxxx, false -: : +- SubqueryAlias t2 -: : +- Project [(n + 1)#x AS j#x] -: : +- Project [(n#x + 1) AS (n + 1)#x] -: : +- SubqueryAlias t1 -: : +- Project [col1#x AS n#x] -: : +- UnionLoopRef xxxx, [col1#x], false -: :- CTERelationDef xxxx, false -: : +- SubqueryAlias t3 -: : +- Project [j#x AS k#x] -: : +- Project [j#x] -: : +- SubqueryAlias t2 -: : +- CTERelationRef xxxx, true, [j#x], false, false -: +- Project [k#x] -: +- Filter (k#x <= 5) -: +- SubqueryAlias t3 -: +- CTERelationRef xxxx, true, [k#x], false, false +: +- UnionLoop xxxx +: :- Project [col1#x AS n#x] +: : +- LocalRelation [col1#x] +: +- WithCTE +: :- CTERelationDef xxxx, false +: : +- SubqueryAlias t2 +: : +- Project [(n + 1)#x AS j#x] +: : +- Project [(n#x + 1) AS (n + 1)#x] +: : +- SubqueryAlias t1 +: : +- UnionLoopRef xxxx, [n#x], false +: :- CTERelationDef xxxx, false +: : +- SubqueryAlias t3 +: : +- Project [j#x AS k#x] +: : +- Project [j#x] +: : +- SubqueryAlias t2 +: : +- CTERelationRef xxxx, true, [j#x], false, false +: +- Project [k#x] +: +- Filter (k#x <= 5) +: +- SubqueryAlias t3 +: +- CTERelationRef xxxx, true, [k#x], false, false +- Project [n#x] +- SubqueryAlias t1 +- CTERelationRef xxxx, true, [n#x], false, false @@ -1544,26 +1515,25 @@ SELECT * FROM r2 WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias r2 -: +- Project [outerlevel#x AS outerlevel1#x, innerlevel#x AS innerlevel1#x] -: +- WithCTE -: :- CTERelationDef xxxx, false -: : +- SubqueryAlias r1 -: : +- UnionLoop xxxx -: : :- Project [0 AS innerlevel#x] -: : : +- OneRowRelation -: : +- Project [(innerlevel#x + 1) AS (innerlevel + 1)#x] -: : +- Filter (innerlevel#x < 3) -: : +- SubqueryAlias r1 -: : +- UnionLoopRef xxxx, [innerlevel#x], false -: +- UnionLoop xxxx -: :- Project [0 AS outerlevel#x, innerlevel#x] -: : +- SubqueryAlias r1 -: : +- CTERelationRef xxxx, true, [innerlevel#x], false, false -: +- Project [(outerlevel1#x + 1) AS (outerlevel1 + 1)#x, innerlevel1#x] -: +- Filter (outerlevel1#x < 3) -: +- SubqueryAlias r2 -: +- Project [outerlevel#x AS outerlevel1#x, innerlevel#x AS innerlevel1#x] -: +- UnionLoopRef xxxx, [outerlevel#x, innerlevel#x], false +: +- WithCTE +: :- CTERelationDef xxxx, false +: : +- SubqueryAlias r1 +: : +- UnionLoop xxxx +: : :- Project [0 AS innerlevel#x] +: : : +- OneRowRelation +: : +- Project [(innerlevel#x + 1) AS (innerlevel + 1)#x] +: : +- Filter (innerlevel#x < 3) +: : +- SubqueryAlias r1 +: : +- UnionLoopRef xxxx, [innerlevel#x], false +: +- UnionLoop xxxx +: :- Project [outerlevel#x AS outerlevel1#x, innerlevel#x AS innerlevel1#x] +: : +- Project [0 AS outerlevel#x, innerlevel#x] +: : +- SubqueryAlias r1 +: : +- CTERelationRef xxxx, true, [innerlevel#x], false, false +: +- Project [(outerlevel1#x + 1) AS (outerlevel1 + 1)#x, innerlevel1#x] +: +- Filter (outerlevel1#x < 3) +: +- SubqueryAlias r2 +: +- UnionLoopRef xxxx, [outerlevel1#x, innerlevel1#x], false +- Project [outerlevel1#x, innerlevel1#x] +- SubqueryAlias r2 +- CTERelationRef xxxx, true, [outerlevel1#x, innerlevel1#x], false, false @@ -1580,22 +1550,21 @@ SELECT * FROM t1 WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias t1 -: +- Project [1#x AS n#x] -: +- WithCTE -: :- CTERelationDef xxxx, false -: : +- SubqueryAlias t2 -: : +- Project [n#x AS n#x] -: : +- Project [n#x] -: : +- SubqueryAlias t1 -: : +- Project [1#x AS n#x] -: : +- UnionLoopRef xxxx, [1#x], false -: +- UnionLoop xxxx -: :- Project [1 AS 1#x] -: : +- OneRowRelation -: +- Project [(n#x + 1) AS (n + 1)#x] -: +- Filter (n#x < 5) -: +- SubqueryAlias t2 -: +- CTERelationRef xxxx, true, [n#x], false, false +: +- WithCTE +: :- CTERelationDef xxxx, false +: : +- SubqueryAlias t2 +: : +- Project [n#x AS n#x] +: : +- Project [n#x] +: : +- SubqueryAlias t1 +: : +- UnionLoopRef xxxx, [n#x], false +: +- UnionLoop xxxx +: :- Project [1#x AS n#x] +: : +- Project [1 AS 1#x] +: : +- OneRowRelation +: +- Project [(n#x + 1) AS (n + 1)#x] +: +- Filter (n#x < 5) +: +- SubqueryAlias t2 +: +- CTERelationRef xxxx, true, [n#x], false, false +- Project [n#x] +- SubqueryAlias t1 +- CTERelationRef xxxx, true, [n#x], false, false @@ -1629,3 +1598,67 @@ WithCTE +- Project [n#x] +- SubqueryAlias t1 +- CTERelationRef xxxx, true, [n#x], false, false + + +-- !query +WITH RECURSIVE tmp(x) AS ( + values (1), (2), (3), (4), (5) +), rcte(x, y) AS ( + SELECT x, x FROM tmp WHERE x = 1 + UNION ALL + SELECT x + 1, x FROM rcte WHERE x < 5 +) +SELECT * FROM rcte +-- !query analysis +WithCTE +:- CTERelationDef xxxx, false +: +- SubqueryAlias tmp +: +- Project [col1#x AS x#x] +: +- LocalRelation [col1#x] +:- CTERelationDef xxxx, false +: +- SubqueryAlias rcte +: +- UnionLoop xxxx +: :- Project [x#x AS x#x, x#x AS y#x] +: : +- Project [x#x, x#x] +: : +- Filter (x#x = 1) +: : +- SubqueryAlias tmp +: : +- CTERelationRef xxxx, true, [x#x], false, false, 5 +: +- Project [(x#x + 1) AS (x + 1)#x, x#x] +: +- Filter (x#x < 5) +: +- SubqueryAlias rcte +: +- UnionLoopRef xxxx, [x#x, y#x], false ++- Project [x#x, y#x] + +- SubqueryAlias rcte + +- CTERelationRef xxxx, true, [x#x, y#x], false, false + + +-- !query +WITH RECURSIVE tmp(x) AS ( + values (1), (2), (3), (4), (5) +), rcte(x, y, z, t) AS ( + SELECT x, x, x, x FROM tmp WHERE x = 1 + UNION ALL + SELECT x + 1, x, y + 1, y FROM rcte WHERE x < 5 +) +SELECT * FROM rcte +-- !query analysis +WithCTE +:- CTERelationDef xxxx, false +: +- SubqueryAlias tmp +: +- Project [col1#x AS x#x] +: +- LocalRelation [col1#x] +:- CTERelationDef xxxx, false +: +- SubqueryAlias rcte +: +- UnionLoop xxxx +: :- Project [x#x AS x#x, x#x AS y#x, x#x AS z#x, x#x AS t#x] +: : +- Project [x#x, x#x, x#x, x#x] +: : +- Filter (x#x = 1) +: : +- SubqueryAlias tmp +: : +- CTERelationRef xxxx, true, [x#x], false, false, 5 +: +- Project [(x#x + 1) AS (x + 1)#x, x#x, (y#x + 1) AS (y + 1)#x, y#x] +: +- Filter (x#x < 5) +: +- SubqueryAlias rcte +: +- UnionLoopRef xxxx, [x#x, y#x, z#x, t#x], false ++- Project [x#x, y#x, z#x, t#x] + +- SubqueryAlias rcte + +- CTERelationRef xxxx, true, [x#x, y#x, z#x, t#x], false, false diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/postgreSQL/with.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/postgreSQL/with.sql.out index de86c88f6d1b9..1a8ac8ba846af 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/postgreSQL/with.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/postgreSQL/with.sql.out @@ -40,14 +40,13 @@ SELECT sum(n) FROM t WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias t -: +- Project [col1#x AS n#x] -: +- UnionLoop xxxx -: :- LocalRelation [col1#x] -: +- Project [(n#x + 1) AS (n + 1)#x] -: +- Filter (n#x < 100) -: +- SubqueryAlias t -: +- Project [col1#x AS n#x] -: +- UnionLoopRef xxxx, [col1#x], false +: +- UnionLoop xxxx +: :- Project [col1#x AS n#x] +: : +- LocalRelation [col1#x] +: +- Project [(n#x + 1) AS (n + 1)#x] +: +- Filter (n#x < 100) +: +- SubqueryAlias t +: +- UnionLoopRef xxxx, [n#x], false +- Aggregate [sum(n#x) AS sum(n)#xL] +- SubqueryAlias t +- CTERelationRef xxxx, true, [n#x], false, false @@ -64,16 +63,15 @@ SELECT * FROM t WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias t -: +- Project [scalarsubquery()#x AS n#x] -: +- UnionLoop xxxx -: :- Project [scalar-subquery#x [] AS scalarsubquery()#x] -: : : +- LocalRelation [col1#x] -: : +- OneRowRelation -: +- Project [(n#x + 1) AS (n + 1)#x] -: +- Filter (n#x < 5) -: +- SubqueryAlias t -: +- Project [scalarsubquery()#x AS n#x] -: +- UnionLoopRef xxxx, [scalarsubquery()#x], false +: +- UnionLoop xxxx +: :- Project [scalarsubquery()#x AS n#x] +: : +- Project [scalar-subquery#x [] AS scalarsubquery()#x] +: : : +- LocalRelation [col1#x] +: : +- OneRowRelation +: +- Project [(n#x + 1) AS (n + 1)#x] +: +- Filter (n#x < 5) +: +- SubqueryAlias t +: +- UnionLoopRef xxxx, [n#x], false +- Project [n#x] +- SubqueryAlias t +- CTERelationRef xxxx, true, [n#x], false, false @@ -97,14 +95,13 @@ SELECT * FROM nums, false, false, LocalTempView, UNSUPPORTED, true +- WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias nums - : +- Project [col1#x AS n#x] - : +- UnionLoop xxxx - : :- LocalRelation [col1#x] - : +- Project [(n#x + 1) AS (n + 1)#x] - : +- Filter (n#x < 5) - : +- SubqueryAlias nums - : +- Project [col1#x AS n#x] - : +- UnionLoopRef xxxx, [col1#x], false + : +- UnionLoop xxxx + : :- Project [col1#x AS n#x] + : : +- LocalRelation [col1#x] + : +- Project [(n#x + 1) AS (n + 1)#x] + : +- Filter (n#x < 5) + : +- SubqueryAlias nums + : +- UnionLoopRef xxxx, [n#x], false +- Project [n#x] +- SubqueryAlias nums +- CTERelationRef xxxx, true, [n#x], false, false @@ -120,14 +117,13 @@ Project [n#x] +- WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias nums - : +- Project [col1#x AS n#x] - : +- UnionLoop xxxx - : :- LocalRelation [col1#x] - : +- Project [(n#x + 1) AS (n + 1)#x] - : +- Filter (n#x < 5) - : +- SubqueryAlias nums - : +- Project [col1#x AS n#x] - : +- UnionLoopRef xxxx, [col1#x], false + : +- UnionLoop xxxx + : :- Project [col1#x AS n#x] + : : +- LocalRelation [col1#x] + : +- Project [(n#x + 1) AS (n + 1)#x] + : +- Filter (n#x < 5) + : +- SubqueryAlias nums + : +- UnionLoopRef xxxx, [n#x], false +- Project [n#x] +- SubqueryAlias nums +- CTERelationRef xxxx, true, [n#x], false, false @@ -151,14 +147,13 @@ SELECT * FROM nums, false, true, LocalTempView, UNSUPPORTED, true +- WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias nums - : +- Project [col1#x AS n#x] - : +- UnionLoop xxxx - : :- LocalRelation [col1#x] - : +- Project [(n#x + 1) AS (n + 1)#x] - : +- Filter (n#x < 6) - : +- SubqueryAlias nums - : +- Project [col1#x AS n#x] - : +- UnionLoopRef xxxx, [col1#x], false + : +- UnionLoop xxxx + : :- Project [col1#x AS n#x] + : : +- LocalRelation [col1#x] + : +- Project [(n#x + 1) AS (n + 1)#x] + : +- Filter (n#x < 6) + : +- SubqueryAlias nums + : +- UnionLoopRef xxxx, [n#x], false +- Project [n#x] +- SubqueryAlias nums +- CTERelationRef xxxx, true, [n#x], false, false @@ -174,14 +169,13 @@ Project [n#x] +- WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias nums - : +- Project [col1#x AS n#x] - : +- UnionLoop xxxx - : :- LocalRelation [col1#x] - : +- Project [(n#x + 1) AS (n + 1)#x] - : +- Filter (n#x < 6) - : +- SubqueryAlias nums - : +- Project [col1#x AS n#x] - : +- UnionLoopRef xxxx, [col1#x], false + : +- UnionLoop xxxx + : :- Project [col1#x AS n#x] + : : +- LocalRelation [col1#x] + : +- Project [(n#x + 1) AS (n + 1)#x] + : +- Filter (n#x < 6) + : +- SubqueryAlias nums + : +- UnionLoopRef xxxx, [n#x], false +- Project [n#x] +- SubqueryAlias nums +- CTERelationRef xxxx, true, [n#x], false, false @@ -218,13 +212,12 @@ SELECT * FROM t LIMIT 10 WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias t -: +- Project [col1#x AS n#x] -: +- UnionLoop xxxx -: :- LocalRelation [col1#x] -: +- Project [(n#x + 1) AS (n + 1)#x] -: +- SubqueryAlias t -: +- Project [col1#x AS n#x] -: +- UnionLoopRef xxxx, [col1#x], false +: +- UnionLoop xxxx +: :- Project [col1#x AS n#x] +: : +- LocalRelation [col1#x] +: +- Project [(n#x + 1) AS (n + 1)#x] +: +- SubqueryAlias t +: +- UnionLoopRef xxxx, [n#x], false +- GlobalLimit 10 +- LocalLimit 10 +- Project [n#x] @@ -278,15 +271,14 @@ SELECT n AS is_text FROM t WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias t -: +- Project [foo#x AS n#x] -: +- UnionLoop xxxx -: :- Project [foo AS foo#x] -: : +- OneRowRelation -: +- Project [concat(n#x, bar) AS concat(n, bar)#x] -: +- Filter (length(n#x) < 20) -: +- SubqueryAlias t -: +- Project [foo#x AS n#x] -: +- UnionLoopRef xxxx, [foo#x], false +: +- UnionLoop xxxx +: :- Project [foo#x AS n#x] +: : +- Project [foo AS foo#x] +: : +- OneRowRelation +: +- Project [concat(n#x, bar) AS concat(n, bar)#x] +: +- Filter (length(n#x) < 20) +: +- SubqueryAlias t +: +- UnionLoopRef xxxx, [n#x], false +- Project [n#x AS is_text#x] +- SubqueryAlias t +- CTERelationRef xxxx, true, [n#x], false, false @@ -435,22 +427,21 @@ SELECT * FROM subdepartment ORDER BY name WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias subdepartment -: +- Project [1#x AS level#x, id#x AS id#x, parent_department#x AS parent_department#x, name#x AS name#x] -: +- UnionLoop xxxx -: :- Project [1 AS 1#x, id#x, parent_department#x, name#x] -: : +- Filter (name#x = A) -: : +- SubqueryAlias spark_catalog.default.department -: : +- Relation spark_catalog.default.department[id#x,parent_department#x,name#x] parquet -: +- Project [(level#x + 1) AS (level + 1)#x, id#x, parent_department#x, name#x] -: +- Filter (parent_department#x = id#x) -: +- Join Inner -: :- SubqueryAlias d -: : +- SubqueryAlias spark_catalog.default.department -: : +- Relation spark_catalog.default.department[id#x,parent_department#x,name#x] parquet -: +- SubqueryAlias sd -: +- SubqueryAlias subdepartment -: +- Project [1#x AS level#x, id#x AS id#x, parent_department#x AS parent_department#x, name#x AS name#x] -: +- UnionLoopRef xxxx, [1#x, id#x, parent_department#x, name#x], false +: +- UnionLoop xxxx +: :- Project [1#x AS level#x, id#x AS id#x, parent_department#x AS parent_department#x, name#x AS name#x] +: : +- Project [1 AS 1#x, id#x, parent_department#x, name#x] +: : +- Filter (name#x = A) +: : +- SubqueryAlias spark_catalog.default.department +: : +- Relation spark_catalog.default.department[id#x,parent_department#x,name#x] parquet +: +- Project [(level#x + 1) AS (level + 1)#x, id#x, parent_department#x, name#x] +: +- Filter (parent_department#x = id#x) +: +- Join Inner +: :- SubqueryAlias d +: : +- SubqueryAlias spark_catalog.default.department +: : +- Relation spark_catalog.default.department[id#x,parent_department#x,name#x] parquet +: +- SubqueryAlias sd +: +- SubqueryAlias subdepartment +: +- UnionLoopRef xxxx, [level#x, id#x, parent_department#x, name#x], false +- Sort [name#x ASC NULLS FIRST], true +- Project [level#x, id#x, parent_department#x, name#x] +- SubqueryAlias subdepartment @@ -472,22 +463,21 @@ SELECT * FROM subdepartment WHERE level >= 2 ORDER BY name WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias subdepartment -: +- Project [1#x AS level#x, id#x AS id#x, parent_department#x AS parent_department#x, name#x AS name#x] -: +- UnionLoop xxxx -: :- Project [1 AS 1#x, id#x, parent_department#x, name#x] -: : +- Filter (name#x = A) -: : +- SubqueryAlias spark_catalog.default.department -: : +- Relation spark_catalog.default.department[id#x,parent_department#x,name#x] parquet -: +- Project [(level#x + 1) AS (level + 1)#x, id#x, parent_department#x, name#x] -: +- Filter (parent_department#x = id#x) -: +- Join Inner -: :- SubqueryAlias d -: : +- SubqueryAlias spark_catalog.default.department -: : +- Relation spark_catalog.default.department[id#x,parent_department#x,name#x] parquet -: +- SubqueryAlias sd -: +- SubqueryAlias subdepartment -: +- Project [1#x AS level#x, id#x AS id#x, parent_department#x AS parent_department#x, name#x AS name#x] -: +- UnionLoopRef xxxx, [1#x, id#x, parent_department#x, name#x], false +: +- UnionLoop xxxx +: :- Project [1#x AS level#x, id#x AS id#x, parent_department#x AS parent_department#x, name#x AS name#x] +: : +- Project [1 AS 1#x, id#x, parent_department#x, name#x] +: : +- Filter (name#x = A) +: : +- SubqueryAlias spark_catalog.default.department +: : +- Relation spark_catalog.default.department[id#x,parent_department#x,name#x] parquet +: +- Project [(level#x + 1) AS (level + 1)#x, id#x, parent_department#x, name#x] +: +- Filter (parent_department#x = id#x) +: +- Join Inner +: :- SubqueryAlias d +: : +- SubqueryAlias spark_catalog.default.department +: : +- Relation spark_catalog.default.department[id#x,parent_department#x,name#x] parquet +: +- SubqueryAlias sd +: +- SubqueryAlias subdepartment +: +- UnionLoopRef xxxx, [level#x, id#x, parent_department#x, name#x], false +- Sort [name#x ASC NULLS FIRST], true +- Project [level#x, id#x, parent_department#x, name#x] +- Filter (level#x >= 2) @@ -542,15 +532,14 @@ Aggregate [count(1) AS count(1)#xL] : +- WithCTE : :- CTERelationDef xxxx, false : : +- SubqueryAlias t - : : +- Project [1#x AS n#x] - : : +- UnionLoop xxxx - : : :- Project [1 AS 1#x] - : : : +- OneRowRelation - : : +- Project [(n#x + 1) AS (n + 1)#x] - : : +- Filter (n#x < 100) - : : +- SubqueryAlias t - : : +- Project [1#x AS n#x] - : : +- UnionLoopRef xxxx, [1#x], false + : : +- UnionLoop xxxx + : : :- Project [1#x AS n#x] + : : : +- Project [1 AS 1#x] + : : : +- OneRowRelation + : : +- Project [(n#x + 1) AS (n + 1)#x] + : : +- Filter (n#x < 100) + : : +- SubqueryAlias t + : : +- UnionLoopRef xxxx, [n#x], false : +- Project [n#x] : +- Filter (n#x < 50000) : +- SubqueryAlias t @@ -559,15 +548,14 @@ Aggregate [count(1) AS count(1)#xL] +- WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias t - : +- Project [1#x AS n#x] - : +- UnionLoop xxxx - : :- Project [1 AS 1#x] - : : +- OneRowRelation - : +- Project [(n#x + 1) AS (n + 1)#x] - : +- Filter (n#x < 200) - : +- SubqueryAlias t - : +- Project [1#x AS n#x] - : +- UnionLoopRef xxxx, [1#x], false + : +- UnionLoop xxxx + : :- Project [1#x AS n#x] + : : +- Project [1 AS 1#x] + : : +- OneRowRelation + : +- Project [(n#x + 1) AS (n + 1)#x] + : +- Filter (n#x < 200) + : +- SubqueryAlias t + : +- UnionLoopRef xxxx, [n#x], false +- Project [n#x] +- SubqueryAlias t +- CTERelationRef xxxx, true, [n#x], false, false @@ -691,14 +679,13 @@ SELECT sum(n) AS sum FROM t, false, false, PersistedView, COMPENSATION, true +- WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias t - : +- Project [col1#x AS n#x] - : +- UnionLoop xxxx - : :- LocalRelation [col1#x] - : +- Project [(n#x + 1) AS (n + 1)#x] - : +- Filter (n#x < 100) - : +- SubqueryAlias t - : +- Project [col1#x AS n#x] - : +- UnionLoopRef xxxx, [col1#x], false + : +- UnionLoop xxxx + : :- Project [col1#x AS n#x] + : : +- LocalRelation [col1#x] + : +- Project [(n#x + 1) AS (n + 1)#x] + : +- Filter (n#x < 100) + : +- SubqueryAlias t + : +- UnionLoopRef xxxx, [n#x], false +- Aggregate [sum(n#x) AS sum#xL] +- SubqueryAlias t +- CTERelationRef xxxx, true, [n#x], false, false @@ -714,14 +701,13 @@ Project [sum#xL] +- WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias t - : +- Project [col1#x AS n#x] - : +- UnionLoop xxxx - : :- LocalRelation [col1#x] - : +- Project [(n#x + 1) AS (n + 1)#x] - : +- Filter (n#x < 100) - : +- SubqueryAlias t - : +- Project [col1#x AS n#x] - : +- UnionLoopRef xxxx, [col1#x], false + : +- UnionLoop xxxx + : :- Project [col1#x AS n#x] + : : +- LocalRelation [col1#x] + : +- Project [(n#x + 1) AS (n + 1)#x] + : +- Filter (n#x < 100) + : +- SubqueryAlias t + : +- UnionLoopRef xxxx, [n#x], false +- Aggregate [sum(n#x) AS sum#xL] +- SubqueryAlias t +- CTERelationRef xxxx, true, [n#x], false, false @@ -740,20 +726,19 @@ WITH RECURSIVE t(i,j) AS ( WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias t -: +- Project [col1#x AS i#x, col2#x AS j#x] -: +- UnionLoop xxxx -: :- LocalRelation [col1#x, col2#x] -: +- Project [i#x, (j#x + 1) AS (j + 1)#x] -: +- Join Inner, (i#x = (i#x + 1)) -: :- SubqueryAlias t2 -: : +- Union false, false -: : :- Project [2 AS i#x] -: : : +- OneRowRelation -: : +- Project [3 AS i#x] -: : +- OneRowRelation -: +- SubqueryAlias t -: +- Project [col1#x AS i#x, col2#x AS j#x] -: +- UnionLoopRef xxxx, [col1#x, col2#x], false +: +- UnionLoop xxxx +: :- Project [col1#x AS i#x, col2#x AS j#x] +: : +- LocalRelation [col1#x, col2#x] +: +- Project [i#x, (j#x + 1) AS (j + 1)#x] +: +- Join Inner, (i#x = (i#x + 1)) +: :- SubqueryAlias t2 +: : +- Union false, false +: : :- Project [2 AS i#x] +: : : +- OneRowRelation +: : +- Project [3 AS i#x] +: : +- OneRowRelation +: +- SubqueryAlias t +: +- UnionLoopRef xxxx, [i#x, j#x], false +- Project [i#x, j#x] +- SubqueryAlias t +- CTERelationRef xxxx, true, [i#x, j#x], false, false @@ -794,16 +779,15 @@ SELECT t1.*, t2.* FROM t AS t1 JOIN t AS t2 ON WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias t -: +- Project [col1#x AS id#x, col2#x AS path#x] -: +- UnionLoop xxxx -: :- LocalRelation [col1#x, col2#x] -: +- Project [id#x, concat(path#x, array(id#x)) AS concat(path, array(id))#x] -: +- Join Inner, (parent_id#x = id#x) -: :- SubqueryAlias spark_catalog.default.tree -: : +- Relation spark_catalog.default.tree[id#x,parent_id#x] parquet -: +- SubqueryAlias t -: +- Project [col1#x AS id#x, col2#x AS path#x] -: +- UnionLoopRef xxxx, [col1#x, col2#x], false +: +- UnionLoop xxxx +: :- Project [col1#x AS id#x, col2#x AS path#x] +: : +- LocalRelation [col1#x, col2#x] +: +- Project [id#x, concat(path#x, array(id#x)) AS concat(path, array(id))#x] +: +- Join Inner, (parent_id#x = id#x) +: :- SubqueryAlias spark_catalog.default.tree +: : +- Relation spark_catalog.default.tree[id#x,parent_id#x] parquet +: +- SubqueryAlias t +: +- UnionLoopRef xxxx, [id#x, path#x], false +- Sort [id#x ASC NULLS FIRST, id#x ASC NULLS FIRST], true +- Project [id#x, path#x, id#x, path#x] +- Join Inner, (((path#x[0] = path#x[0]) AND (size(path#x, false) = 1)) AND (size(path#x, false) > 1)) @@ -832,16 +816,15 @@ SELECT t1.id, count(*) FROM t AS t1 JOIN t AS t2 ON WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias t -: +- Project [col1#x AS id#x, col2#x AS path#x] -: +- UnionLoop xxxx -: :- LocalRelation [col1#x, col2#x] -: +- Project [id#x, concat(path#x, array(id#x)) AS concat(path, array(id))#x] -: +- Join Inner, (parent_id#x = id#x) -: :- SubqueryAlias spark_catalog.default.tree -: : +- Relation spark_catalog.default.tree[id#x,parent_id#x] parquet -: +- SubqueryAlias t -: +- Project [col1#x AS id#x, col2#x AS path#x] -: +- UnionLoopRef xxxx, [col1#x, col2#x], false +: +- UnionLoop xxxx +: :- Project [col1#x AS id#x, col2#x AS path#x] +: : +- LocalRelation [col1#x, col2#x] +: +- Project [id#x, concat(path#x, array(id#x)) AS concat(path, array(id))#x] +: +- Join Inner, (parent_id#x = id#x) +: :- SubqueryAlias spark_catalog.default.tree +: : +- Relation spark_catalog.default.tree[id#x,parent_id#x] parquet +: +- SubqueryAlias t +: +- UnionLoopRef xxxx, [id#x, path#x], false +- Sort [id#x ASC NULLS FIRST], true +- Aggregate [id#x], [id#x, count(1) AS count(1)#xL] +- Join Inner, (((path#x[0] = path#x[0]) AND (size(path#x, false) = 1)) AND (size(path#x, false) > 1)) @@ -866,16 +849,15 @@ SELECT t1.id, t2.path, struct(t2.*) FROM t AS t1 JOIN t AS t2 ON WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias t -: +- Project [col1#x AS id#x, col2#x AS path#x] -: +- UnionLoop xxxx -: :- LocalRelation [col1#x, col2#x] -: +- Project [id#x, concat(path#x, array(id#x)) AS concat(path, array(id))#x] -: +- Join Inner, (parent_id#x = id#x) -: :- SubqueryAlias spark_catalog.default.tree -: : +- Relation spark_catalog.default.tree[id#x,parent_id#x] parquet -: +- SubqueryAlias t -: +- Project [col1#x AS id#x, col2#x AS path#x] -: +- UnionLoopRef xxxx, [col1#x, col2#x], false +: +- UnionLoop xxxx +: :- Project [col1#x AS id#x, col2#x AS path#x] +: : +- LocalRelation [col1#x, col2#x] +: +- Project [id#x, concat(path#x, array(id#x)) AS concat(path, array(id))#x] +: +- Join Inner, (parent_id#x = id#x) +: :- SubqueryAlias spark_catalog.default.tree +: : +- Relation spark_catalog.default.tree[id#x,parent_id#x] parquet +: +- SubqueryAlias t +: +- UnionLoopRef xxxx, [id#x, path#x], false +- Project [id#x, path#x, struct(id, id#x, path, path#x) AS struct(id, path)#x] +- Join Inner, (id#x = id#x) :- SubqueryAlias t1 @@ -919,22 +901,21 @@ select * from search_graph WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias search_graph -: +- Project [f#x AS f#x, t#x AS t#x, label#x AS label#x, array(struct(f, t))#x AS path#x, false#x AS cycle#x] -: +- UnionLoop xxxx -: :- Project [f#x, t#x, label#x, array(struct(f, f#x, t, t#x)) AS array(struct(f, t))#x, false AS false#x] -: : +- SubqueryAlias g -: : +- SubqueryAlias spark_catalog.default.graph -: : +- Relation spark_catalog.default.graph[f#x,t#x,label#x] parquet -: +- Project [f#x, t#x, label#x, concat(path#x, array(struct(f, f#x, t, t#x))) AS concat(path, array(struct(f, t)))#x, array_contains(path#x, struct(f, f#x, t, t#x)) AS array_contains(path, struct(f, t))#x] -: +- Filter ((f#x = t#x) AND NOT cycle#x) -: +- Join Inner -: :- SubqueryAlias g -: : +- SubqueryAlias spark_catalog.default.graph -: : +- Relation spark_catalog.default.graph[f#x,t#x,label#x] parquet -: +- SubqueryAlias sg -: +- SubqueryAlias search_graph -: +- Project [f#x AS f#x, t#x AS t#x, label#x AS label#x, array(struct(f, t))#x AS path#x, false#x AS cycle#x] -: +- UnionLoopRef xxxx, [f#x, t#x, label#x, array(struct(f, t))#x, false#x], false +: +- UnionLoop xxxx +: :- Project [f#x AS f#x, t#x AS t#x, label#x AS label#x, array(struct(f, t))#x AS path#x, false#x AS cycle#x] +: : +- Project [f#x, t#x, label#x, array(struct(f, f#x, t, t#x)) AS array(struct(f, t))#x, false AS false#x] +: : +- SubqueryAlias g +: : +- SubqueryAlias spark_catalog.default.graph +: : +- Relation spark_catalog.default.graph[f#x,t#x,label#x] parquet +: +- Project [f#x, t#x, label#x, concat(path#x, array(struct(f, f#x, t, t#x))) AS concat(path, array(struct(f, t)))#x, array_contains(path#x, struct(f, f#x, t, t#x)) AS array_contains(path, struct(f, t))#x] +: +- Filter ((f#x = t#x) AND NOT cycle#x) +: +- Join Inner +: :- SubqueryAlias g +: : +- SubqueryAlias spark_catalog.default.graph +: : +- Relation spark_catalog.default.graph[f#x,t#x,label#x] parquet +: +- SubqueryAlias sg +: +- SubqueryAlias search_graph +: +- UnionLoopRef xxxx, [f#x, t#x, label#x, path#x, cycle#x], false +- Project [f#x, t#x, label#x, path#x, cycle#x] +- SubqueryAlias search_graph +- CTERelationRef xxxx, true, [f#x, t#x, label#x, path#x, cycle#x], false, false @@ -953,22 +934,21 @@ select * from search_graph order by path WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias search_graph -: +- Project [f#x AS f#x, t#x AS t#x, label#x AS label#x, array(struct(f, t))#x AS path#x, false#x AS cycle#x] -: +- UnionLoop xxxx -: :- Project [f#x, t#x, label#x, array(struct(f, f#x, t, t#x)) AS array(struct(f, t))#x, false AS false#x] -: : +- SubqueryAlias g -: : +- SubqueryAlias spark_catalog.default.graph -: : +- Relation spark_catalog.default.graph[f#x,t#x,label#x] parquet -: +- Project [f#x, t#x, label#x, concat(path#x, array(struct(f, f#x, t, t#x))) AS concat(path, array(struct(f, t)))#x, array_contains(path#x, struct(f, f#x, t, t#x)) AS array_contains(path, struct(f, t))#x] -: +- Filter ((f#x = t#x) AND NOT cycle#x) -: +- Join Inner -: :- SubqueryAlias g -: : +- SubqueryAlias spark_catalog.default.graph -: : +- Relation spark_catalog.default.graph[f#x,t#x,label#x] parquet -: +- SubqueryAlias sg -: +- SubqueryAlias search_graph -: +- Project [f#x AS f#x, t#x AS t#x, label#x AS label#x, array(struct(f, t))#x AS path#x, false#x AS cycle#x] -: +- UnionLoopRef xxxx, [f#x, t#x, label#x, array(struct(f, t))#x, false#x], false +: +- UnionLoop xxxx +: :- Project [f#x AS f#x, t#x AS t#x, label#x AS label#x, array(struct(f, t))#x AS path#x, false#x AS cycle#x] +: : +- Project [f#x, t#x, label#x, array(struct(f, f#x, t, t#x)) AS array(struct(f, t))#x, false AS false#x] +: : +- SubqueryAlias g +: : +- SubqueryAlias spark_catalog.default.graph +: : +- Relation spark_catalog.default.graph[f#x,t#x,label#x] parquet +: +- Project [f#x, t#x, label#x, concat(path#x, array(struct(f, f#x, t, t#x))) AS concat(path, array(struct(f, t)))#x, array_contains(path#x, struct(f, f#x, t, t#x)) AS array_contains(path, struct(f, t))#x] +: +- Filter ((f#x = t#x) AND NOT cycle#x) +: +- Join Inner +: :- SubqueryAlias g +: : +- SubqueryAlias spark_catalog.default.graph +: : +- Relation spark_catalog.default.graph[f#x,t#x,label#x] parquet +: +- SubqueryAlias sg +: +- SubqueryAlias search_graph +: +- UnionLoopRef xxxx, [f#x, t#x, label#x, path#x, cycle#x], false +- Sort [path#x ASC NULLS FIRST], true +- Project [f#x, t#x, label#x, path#x, cycle#x] +- SubqueryAlias search_graph @@ -988,16 +968,15 @@ WithCTE : +- LocalRelation [col1#x] :- CTERelationDef xxxx, false : +- SubqueryAlias x -: +- Project [id#x AS id#x] -: +- UnionLoop xxxx -: :- Project [id#x] -: : +- SubqueryAlias y -: : +- CTERelationRef xxxx, true, [id#x], false, false, 1 -: +- Project [(id#x + 1) AS (id + 1)#x] -: +- Filter (id#x < 5) -: +- SubqueryAlias x -: +- Project [id#x AS id#x] -: +- UnionLoopRef xxxx, [id#x], false +: +- UnionLoop xxxx +: :- Project [id#x AS id#x] +: : +- Project [id#x] +: : +- SubqueryAlias y +: : +- CTERelationRef xxxx, true, [id#x], false, false, 1 +: +- Project [(id#x + 1) AS (id + 1)#x] +: +- Filter (id#x < 5) +: +- SubqueryAlias x +: +- UnionLoopRef xxxx, [id#x], false +- Project [id#x] +- SubqueryAlias x +- CTERelationRef xxxx, true, [id#x], false, false @@ -1016,15 +995,14 @@ WITH RECURSIVE WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias x -: +- Project [1#x AS id#x] -: +- UnionLoop xxxx -: :- Project [1 AS 1#x] -: : +- OneRowRelation -: +- Project [(id#x + 1) AS (id + 1)#x] -: +- Filter (id#x < 3) -: +- SubqueryAlias x -: +- Project [1#x AS id#x] -: +- UnionLoopRef xxxx, [1#x], false +: +- UnionLoop xxxx +: :- Project [1#x AS id#x] +: : +- Project [1 AS 1#x] +: : +- OneRowRelation +: +- Project [(id#x + 1) AS (id + 1)#x] +: +- Filter (id#x < 3) +: +- SubqueryAlias x +: +- UnionLoopRef xxxx, [id#x], false :- CTERelationDef xxxx, false : +- SubqueryAlias y : +- Project [id#x AS id#x] @@ -1037,16 +1015,15 @@ WithCTE : +- CTERelationRef xxxx, true, [id#x], false, false :- CTERelationDef xxxx, false : +- SubqueryAlias z -: +- Project [id#x AS id#x] -: +- UnionLoop xxxx -: :- Project [id#x] -: : +- SubqueryAlias x -: : +- CTERelationRef xxxx, true, [id#x], false, false -: +- Project [(id#x + 1) AS (id + 1)#x] -: +- Filter (id#x < 10) -: +- SubqueryAlias z -: +- Project [id#x AS id#x] -: +- UnionLoopRef xxxx, [id#x], false +: +- UnionLoop xxxx +: :- Project [id#x AS id#x] +: : +- Project [id#x] +: : +- SubqueryAlias x +: : +- CTERelationRef xxxx, true, [id#x], false, false +: +- Project [(id#x + 1) AS (id + 1)#x] +: +- Filter (id#x < 10) +: +- SubqueryAlias z +: +- UnionLoopRef xxxx, [id#x], false +- Project [id#x] +- SubqueryAlias z +- CTERelationRef xxxx, true, [id#x], false, false @@ -1065,15 +1042,14 @@ WITH RECURSIVE WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias x -: +- Project [1#x AS id#x] -: +- UnionLoop xxxx -: :- Project [1 AS 1#x] -: : +- OneRowRelation -: +- Project [(id#x + 1) AS (id + 1)#x] -: +- Filter (id#x < 3) -: +- SubqueryAlias x -: +- Project [1#x AS id#x] -: +- UnionLoopRef xxxx, [1#x], false +: +- UnionLoop xxxx +: :- Project [1#x AS id#x] +: : +- Project [1 AS 1#x] +: : +- OneRowRelation +: +- Project [(id#x + 1) AS (id + 1)#x] +: +- Filter (id#x < 3) +: +- SubqueryAlias x +: +- UnionLoopRef xxxx, [id#x], false :- CTERelationDef xxxx, false : +- SubqueryAlias y : +- Project [id#x AS id#x] @@ -1086,16 +1062,15 @@ WithCTE : +- CTERelationRef xxxx, true, [id#x], false, false :- CTERelationDef xxxx, false : +- SubqueryAlias z -: +- Project [id#x AS id#x] -: +- UnionLoop xxxx -: :- Project [id#x] -: : +- SubqueryAlias y -: : +- CTERelationRef xxxx, true, [id#x], false, false -: +- Project [(id#x + 1) AS (id + 1)#x] -: +- Filter (id#x < 10) -: +- SubqueryAlias z -: +- Project [id#x AS id#x] -: +- UnionLoopRef xxxx, [id#x], false +: +- UnionLoop xxxx +: :- Project [id#x AS id#x] +: : +- Project [id#x] +: : +- SubqueryAlias y +: : +- CTERelationRef xxxx, true, [id#x], false, false +: +- Project [(id#x + 1) AS (id + 1)#x] +: +- Filter (id#x < 10) +: +- SubqueryAlias z +: +- UnionLoopRef xxxx, [id#x], false +- Project [id#x] +- SubqueryAlias z +- CTERelationRef xxxx, true, [id#x], false, false @@ -1263,19 +1238,17 @@ WITH RECURSIVE x(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM x WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias x -: +- Project [1#x AS n#x] -: +- UnionLoop xxxx -: :- Project [1 AS 1#x] -: : +- OneRowRelation -: +- Project [(n#x + 1) AS (n + 1)#x] -: +- Filter n#x IN (list#x []) -: : +- Project [n#x] -: : +- SubqueryAlias x -: : +- Project [1#x AS n#x] -: : +- UnionLoopRef xxxx, [1#x], false -: +- SubqueryAlias x -: +- Project [1#x AS n#x] -: +- UnionLoopRef xxxx, [1#x], false +: +- UnionLoop xxxx +: :- Project [1#x AS n#x] +: : +- Project [1 AS 1#x] +: : +- OneRowRelation +: +- Project [(n#x + 1) AS (n + 1)#x] +: +- Filter n#x IN (list#x []) +: : +- Project [n#x] +: : +- SubqueryAlias x +: : +- UnionLoopRef xxxx, [n#x], false +: +- SubqueryAlias x +: +- UnionLoopRef xxxx, [n#x], false +- Project [n#x] +- SubqueryAlias x +- CTERelationRef xxxx, true, [n#x], false, false @@ -1334,18 +1307,16 @@ WITH RECURSIVE x(id) AS (values (1) WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias x -: +- Project [col1#x AS id#x] -: +- UnionLoop xxxx -: :- LocalRelation [col1#x] -: +- Project [scalar-subquery#x [] AS scalarsubquery()#x] -: : +- Project [id#x] -: : +- SubqueryAlias x -: : +- Project [col1#x AS id#x] -: : +- UnionLoopRef xxxx, [col1#x], false -: +- Filter (id#x < 5) -: +- SubqueryAlias x -: +- Project [col1#x AS id#x] -: +- UnionLoopRef xxxx, [col1#x], false +: +- UnionLoop xxxx +: :- Project [col1#x AS id#x] +: : +- LocalRelation [col1#x] +: +- Project [scalar-subquery#x [] AS scalarsubquery()#x] +: : +- Project [id#x] +: : +- SubqueryAlias x +: : +- UnionLoopRef xxxx, [id#x], false +: +- Filter (id#x < 5) +: +- SubqueryAlias x +: +- UnionLoopRef xxxx, [id#x], false +- Project [id#x] +- SubqueryAlias x +- CTERelationRef xxxx, true, [id#x], false, false @@ -1386,20 +1357,18 @@ WITH RECURSIVE foo(i) AS WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias foo -: +- Project [col1#x AS i#x] -: +- UnionLoop xxxx -: :- LocalRelation [col1#x] -: +- Union false, false -: :- Project [(i#x + 1) AS (i + 1)#x] -: : +- Filter (i#x < 10) -: : +- SubqueryAlias foo -: : +- Project [col1#x AS i#x] -: : +- UnionLoopRef xxxx, [col1#x], false -: +- Project [(i#x + 1) AS (i + 1)#x] -: +- Filter (i#x < 5) -: +- SubqueryAlias foo -: +- Project [col1#x AS i#x] -: +- UnionLoopRef xxxx, [col1#x], false +: +- UnionLoop xxxx +: :- Project [col1#x AS i#x] +: : +- LocalRelation [col1#x] +: +- Union false, false +: :- Project [(i#x + 1) AS (i + 1)#x] +: : +- Filter (i#x < 10) +: : +- SubqueryAlias foo +: : +- UnionLoopRef xxxx, [i#x], false +: +- Project [(i#x + 1) AS (i + 1)#x] +: +- Filter (i#x < 5) +: +- SubqueryAlias foo +: +- UnionLoopRef xxxx, [i#x], false +- Project [i#x] +- SubqueryAlias foo +- CTERelationRef xxxx, true, [i#x], false, false @@ -1418,22 +1387,20 @@ WITH RECURSIVE foo(i) AS WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias foo -: +- Project [col1#x AS i#x] -: +- UnionLoop xxxx -: :- LocalRelation [col1#x] -: +- Project [(i + 1)#x] -: +- SubqueryAlias t -: +- Union false, false -: :- Project [(i#x + 1) AS (i + 1)#x] -: : +- Filter (i#x < 10) -: : +- SubqueryAlias foo -: : +- Project [col1#x AS i#x] -: : +- UnionLoopRef xxxx, [col1#x], false -: +- Project [(i#x + 1) AS (i + 1)#x] -: +- Filter (i#x < 5) -: +- SubqueryAlias foo -: +- Project [col1#x AS i#x] -: +- UnionLoopRef xxxx, [col1#x], false +: +- UnionLoop xxxx +: :- Project [col1#x AS i#x] +: : +- LocalRelation [col1#x] +: +- Project [(i + 1)#x] +: +- SubqueryAlias t +: +- Union false, false +: :- Project [(i#x + 1) AS (i + 1)#x] +: : +- Filter (i#x < 10) +: : +- SubqueryAlias foo +: : +- UnionLoopRef xxxx, [i#x], false +: +- Project [(i#x + 1) AS (i + 1)#x] +: +- Filter (i#x < 5) +: +- SubqueryAlias foo +: +- UnionLoopRef xxxx, [i#x], false +- Project [i#x] +- SubqueryAlias foo +- CTERelationRef xxxx, true, [i#x], false, false @@ -1451,20 +1418,18 @@ WITH RECURSIVE foo(i) AS WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias foo -: +- Project [col1#x AS i#x] -: +- UnionLoop xxxx -: :- LocalRelation [col1#x] -: +- Except false -: :- Project [(i#x + 1) AS (i + 1)#x] -: : +- Filter (i#x < 10) -: : +- SubqueryAlias foo -: : +- Project [col1#x AS i#x] -: : +- UnionLoopRef xxxx, [col1#x], false -: +- Project [(i#x + 1) AS (i + 1)#x] -: +- Filter (i#x < 5) -: +- SubqueryAlias foo -: +- Project [col1#x AS i#x] -: +- UnionLoopRef xxxx, [col1#x], false +: +- UnionLoop xxxx +: :- Project [col1#x AS i#x] +: : +- LocalRelation [col1#x] +: +- Except false +: :- Project [(i#x + 1) AS (i + 1)#x] +: : +- Filter (i#x < 10) +: : +- SubqueryAlias foo +: : +- UnionLoopRef xxxx, [i#x], false +: +- Project [(i#x + 1) AS (i + 1)#x] +: +- Filter (i#x < 5) +: +- SubqueryAlias foo +: +- UnionLoopRef xxxx, [i#x], false +- Project [i#x] +- SubqueryAlias foo +- CTERelationRef xxxx, true, [i#x], false, false @@ -1482,20 +1447,18 @@ WITH RECURSIVE foo(i) AS WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias foo -: +- Project [col1#x AS i#x] -: +- UnionLoop xxxx -: :- LocalRelation [col1#x] -: +- Intersect false -: :- Project [(i#x + 1) AS (i + 1)#x] -: : +- Filter (i#x < 10) -: : +- SubqueryAlias foo -: : +- Project [col1#x AS i#x] -: : +- UnionLoopRef xxxx, [col1#x], false -: +- Project [(i#x + 1) AS (i + 1)#x] -: +- Filter (i#x < 5) -: +- SubqueryAlias foo -: +- Project [col1#x AS i#x] -: +- UnionLoopRef xxxx, [col1#x], false +: +- UnionLoop xxxx +: :- Project [col1#x AS i#x] +: : +- LocalRelation [col1#x] +: +- Intersect false +: :- Project [(i#x + 1) AS (i + 1)#x] +: : +- Filter (i#x < 10) +: : +- SubqueryAlias foo +: : +- UnionLoopRef xxxx, [i#x], false +: +- Project [(i#x + 1) AS (i + 1)#x] +: +- Filter (i#x < 5) +: +- SubqueryAlias foo +: +- UnionLoopRef xxxx, [i#x], false +- Project [i#x] +- SubqueryAlias foo +- CTERelationRef xxxx, true, [i#x], false, false @@ -1551,27 +1514,25 @@ SELECT * FROM t WithCTE :- CTERelationDef xxxx, false : +- SubqueryAlias t -: +- Project [i#x AS j#x] -: +- WithCTE -: :- CTERelationDef xxxx, false -: : +- SubqueryAlias s -: : +- Project [col1#x AS i#x] -: : +- UnionLoop xxxx -: : :- LocalRelation [col1#x] -: : +- Project [(i#x + 1) AS (i + 1)#x] -: : +- Filter (i#x < 10) -: : +- SubqueryAlias s -: : +- Project [col1#x AS i#x] -: : +- UnionLoopRef xxxx, [col1#x], false -: +- UnionLoop xxxx -: :- Project [i#x] -: : +- SubqueryAlias s -: : +- CTERelationRef xxxx, true, [i#x], false, false -: +- Project [(j#x + 1) AS (j + 1)#x] -: +- Filter (j#x < 10) -: +- SubqueryAlias t -: +- Project [i#x AS j#x] -: +- UnionLoopRef xxxx, [i#x], false +: +- WithCTE +: :- CTERelationDef xxxx, false +: : +- SubqueryAlias s +: : +- UnionLoop xxxx +: : :- Project [col1#x AS i#x] +: : : +- LocalRelation [col1#x] +: : +- Project [(i#x + 1) AS (i + 1)#x] +: : +- Filter (i#x < 10) +: : +- SubqueryAlias s +: : +- UnionLoopRef xxxx, [i#x], false +: +- UnionLoop xxxx +: :- Project [i#x AS j#x] +: : +- Project [i#x] +: : +- SubqueryAlias s +: : +- CTERelationRef xxxx, true, [i#x], false, false +: +- Project [(j#x + 1) AS (j + 1)#x] +: +- Filter (j#x < 10) +: +- SubqueryAlias t +: +- UnionLoopRef xxxx, [j#x], false +- Project [j#x] +- SubqueryAlias t +- CTERelationRef xxxx, true, [j#x], false, false diff --git a/sql/core/src/test/resources/sql-tests/inputs/cte-recursion.sql b/sql/core/src/test/resources/sql-tests/inputs/cte-recursion.sql index fba8861083be4..73d8253b2ba7f 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/cte-recursion.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/cte-recursion.sql @@ -586,4 +586,24 @@ WITH RECURSIVE t1 AS ( SELECT 1 AS n UNION ALL SELECT n+1 FROM t2 WHERE n < 5) -SELECT * FROM t1; \ No newline at end of file +SELECT * FROM t1; + +-- Recursive CTE with multiple of the same reference in the anchor, which get referenced differently subsequent iterations. +WITH RECURSIVE tmp(x) AS ( + values (1), (2), (3), (4), (5) +), rcte(x, y) AS ( + SELECT x, x FROM tmp WHERE x = 1 + UNION ALL + SELECT x + 1, x FROM rcte WHERE x < 5 +) +SELECT * FROM rcte; + +-- Recursive CTE with multiple of the same reference in the anchor, which get referenced as different variables in subsequent iterations. +WITH RECURSIVE tmp(x) AS ( + values (1), (2), (3), (4), (5) +), rcte(x, y, z, t) AS ( + SELECT x, x, x, x FROM tmp WHERE x = 1 + UNION ALL + SELECT x + 1, x, y + 1, y FROM rcte WHERE x < 5 +) +SELECT * FROM rcte; \ No newline at end of file diff --git a/sql/core/src/test/resources/sql-tests/results/cte-recursion.sql.out b/sql/core/src/test/resources/sql-tests/results/cte-recursion.sql.out index 06f440a3f6335..900ba99c71b1c 100644 --- a/sql/core/src/test/resources/sql-tests/results/cte-recursion.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/cte-recursion.sql.out @@ -1475,3 +1475,41 @@ struct 3 4 5 + + +-- !query +WITH RECURSIVE tmp(x) AS ( + values (1), (2), (3), (4), (5) +), rcte(x, y) AS ( + SELECT x, x FROM tmp WHERE x = 1 + UNION ALL + SELECT x + 1, x FROM rcte WHERE x < 5 +) +SELECT * FROM rcte +-- !query schema +struct +-- !query output +1 1 +2 1 +3 2 +4 3 +5 4 + + +-- !query +WITH RECURSIVE tmp(x) AS ( + values (1), (2), (3), (4), (5) +), rcte(x, y, z, t) AS ( + SELECT x, x, x, x FROM tmp WHERE x = 1 + UNION ALL + SELECT x + 1, x, y + 1, y FROM rcte WHERE x < 5 +) +SELECT * FROM rcte +-- !query schema +struct +-- !query output +1 1 1 1 +2 1 2 1 +3 2 2 1 +4 3 3 2 +5 4 4 3 diff --git a/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out b/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out index af7c4639b8410..abd995e5b4d67 100644 --- a/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out @@ -968,36 +968,35 @@ SELECT * FROM r struct -- !query output == Physical Plan == -* Project (6) -+- UnionLoop (1) - :- LocalRelation (2) - +- Project (5) - +- Filter (4) - +- UnionLoopRef (3) +UnionLoop (1) + :- Project (3) + : +- LocalRelation (2) + +- Project (6) + +- Filter (5) + +- UnionLoopRef (4) (1) UnionLoop Loop id: xxxx -Output [1]: [col1#x] +Output [1]: [level#x] Limit: None (2) LocalRelation Arguments: [col1#x] -(3) UnionLoopRef +(3) Project +Arguments: [col1#x AS level#x] + +(4) UnionLoopRef Loop id: xxxx -Output [1]: [col1#x] +Output [1]: [level#x] Accumulated: false -(4) Filter -Arguments: (col1#x < 9) - -(5) Project -Arguments: [(col1#x + 1) AS (level + 1)#x] +(5) Filter +Arguments: (level#x < 9) -(6) Project [codegen id : 1] -Output [1]: [col1#x AS level#x] -Input [1]: [col1#x] +(6) Project +Arguments: [(level#x + 1) AS (level + 1)#x] -- !query diff --git a/sql/core/src/test/resources/sql-tests/results/explain.sql.out b/sql/core/src/test/resources/sql-tests/results/explain.sql.out index 5719c2620132f..f6d1441bfbf8e 100644 --- a/sql/core/src/test/resources/sql-tests/results/explain.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/explain.sql.out @@ -863,36 +863,35 @@ SELECT * FROM r struct -- !query output == Physical Plan == -* Project (6) -+- UnionLoop (1) - :- LocalRelation (2) - +- Project (5) - +- Filter (4) - +- UnionLoopRef (3) +UnionLoop (1) + :- Project (3) + : +- LocalRelation (2) + +- Project (6) + +- Filter (5) + +- UnionLoopRef (4) (1) UnionLoop Loop id: xxxx -Output [1]: [col1#x] +Output [1]: [level#x] Limit: None (2) LocalRelation Arguments: [col1#x] -(3) UnionLoopRef +(3) Project +Arguments: [col1#x AS level#x] + +(4) UnionLoopRef Loop id: xxxx -Output [1]: [col1#x] +Output [1]: [level#x] Accumulated: false -(4) Filter -Arguments: (col1#x < 9) - -(5) Project -Arguments: [(col1#x + 1) AS (level + 1)#x] +(5) Filter +Arguments: (level#x < 9) -(6) Project [codegen id : 1] -Output [1]: [col1#x AS level#x] -Input [1]: [col1#x] +(6) Project +Arguments: [(level#x + 1) AS (level + 1)#x] -- !query