Skip to content

Commit db23547

Browse files
author
kndrvt
committed
YQL-17269: support PositionalUnionAll for INTERSECT and EXCEPT
commit_hash:4c68311d9e875b6643dd49aae1c385aace4ca978
1 parent a9099f4 commit db23547

File tree

20 files changed

+828
-31
lines changed

20 files changed

+828
-31
lines changed

‎yql/essentials/core/common_opt/yql_co_pgselect.cpp

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -818,10 +818,10 @@ TExprNode::TPtr NormalizeColumnOrder(const TExprNode::TPtr& node, const TColumnO
818818
.Build();
819819
}
820820

821-
TExprNode::TPtr ExpandPositionalUnionAll(const TExprNode& node, const TVector<TColumnOrder>& columnOrders,
821+
TExprNode::TPtr ExpandPositionalSelectOp(const TExprNode& node, const TVector<TColumnOrder>& columnOrders,
822822
TExprNode::TListType children, TExprContext& ctx, TOptimizeContext& optCtx)
823823
{
824-
YQL_ENSURE(node.IsCallable({"UnionAllPositional", "UnionMergePositional"}));
824+
YQL_ENSURE(node.IsCallable({"UnionAllPositional", "UnionMergePositional", "IntersectPositional", "IntersectAllPositional", "ExceptPositional", "ExceptAllPositional"}));
825825
auto targetColumnOrder = optCtx.Types->LookupColumnOrder(node);
826826
YQL_ENSURE(targetColumnOrder);
827827

@@ -831,7 +831,9 @@ TExprNode::TPtr ExpandPositionalUnionAll(const TExprNode& node, const TVector<TC
831831
child = NormalizeColumnOrder(child, childColumnOrder, *targetColumnOrder, ctx);
832832
}
833833

834-
auto res = ctx.NewCallable(node.Pos(), node.IsCallable("UnionAllPositional") ? "UnionAll" : "UnionMerge", std::move(children));
834+
TStringBuf callable = node.Content();
835+
YQL_ENSURE(callable.ChopSuffix("Positional"));
836+
auto res = ctx.NewCallable(node.Pos(), callable, std::move(children));
835837
return KeepColumnOrder(res, node, ctx, *optCtx.Types);
836838
}
837839

‎yql/essentials/core/common_opt/yql_co_pgselect.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ TExprNode::TPtr ExpandPgSelect(const TExprNode::TPtr& node, TExprContext& ctx, T
99
TExprNode::TPtr ExpandPgSelectSublink(const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx,
1010
ui32 subLinkId, const TExprNode::TListType& outerInputs, const TVector<TString>& outerInputAliases);
1111

12-
TExprNode::TPtr ExpandPositionalUnionAll(const TExprNode& input, const TVector<TColumnOrder>& columnOrders,
12+
TExprNode::TPtr ExpandPositionalSelectOp(const TExprNode& input, const TVector<TColumnOrder>& columnOrders,
1313
TExprNode::TListType children, TExprContext& ctx, TOptimizeContext& optCtx);
1414

1515
TExprNode::TPtr NormalizeColumnOrder(const TExprNode::TPtr& node, const TColumnOrder& sourceColumnOrder,

‎yql/essentials/core/common_opt/yql_co_simple1.cpp

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6742,7 +6742,12 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
67426742
return node;
67436743
};
67446744

6745-
map["UnionAllPositional"] = map["UnionMergePositional"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
6745+
map["UnionAllPositional"] = \
6746+
map["UnionMergePositional"] = \
6747+
map["IntersectPositional"] = \
6748+
map["IntersectAllPositional"] = \
6749+
map["ExceptPositional"] = \
6750+
map["ExceptAllPositional"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
67466751
YQL_CLOG(DEBUG, Core) << "Expand " << node->Content();
67476752
if (node->ChildrenSize() == 1) {
67486753
return node->HeadPtr();
@@ -6755,7 +6760,7 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
67556760
columnOrders.push_back(*childColumnOrder);
67566761
}
67576762

6758-
return ExpandPositionalUnionAll(*node, columnOrders, node->ChildrenList(), ctx, optCtx);
6763+
return ExpandPositionalSelectOp(*node, columnOrders, node->ChildrenList(), ctx, optCtx);
67596764
};
67606765

67616766
map["UnionPositional"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) {

‎yql/essentials/core/type_ann/type_ann_core.cpp

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13222,9 +13222,13 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
1322213222
ExtFunctions["JsonVariables"] = &JsonVariablesWrapper;
1322313223
ExtFunctions["AssumeColumnOrder"] = &AssumeColumnOrderWrapper;
1322413224
ExtFunctions["AssumeColumnOrderPartial"] = &AssumeColumnOrderWrapper;
13225-
ExtFunctions["UnionAllPositional"] = &UnionAllPositionalWrapper;
13226-
ExtFunctions["UnionPositional"] = &UnionAllPositionalWrapper;
13227-
ExtFunctions["UnionMergePositional"] = &UnionAllPositionalWrapper;
13225+
ExtFunctions["UnionAllPositional"] = &SelectOpPositionalWrapper;
13226+
ExtFunctions["UnionPositional"] = &SelectOpPositionalWrapper;
13227+
ExtFunctions["UnionMergePositional"] = &SelectOpPositionalWrapper;
13228+
ExtFunctions["IntersectAllPositional"] = &SelectOpPositionalWrapper;
13229+
ExtFunctions["IntersectPositional"] = &SelectOpPositionalWrapper;
13230+
ExtFunctions["ExceptAllPositional"] = &SelectOpPositionalWrapper;
13231+
ExtFunctions["ExceptPositional"] = &SelectOpPositionalWrapper;
1322813232
ExtFunctions["SafeCast"] = &CastWrapper<false>;
1322913233
ExtFunctions["StrictCast"] = &CastWrapper<true>;
1323013234
ExtFunctions["Version"] = &VersionWrapper;

‎yql/essentials/core/type_ann/type_ann_list.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2900,7 +2900,7 @@ namespace {
29002900
return IGraphTransformer::TStatus::Ok;
29012901
}
29022902

2903-
IGraphTransformer::TStatus UnionAllPositionalWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) {
2903+
IGraphTransformer::TStatus SelectOpPositionalWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) {
29042904
Y_UNUSED(output);
29052905
if (!ctx.Types.OrderedColumns) {
29062906
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder()

‎yql/essentials/core/type_ann/type_ann_list.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ namespace NTypeAnnImpl {
6464
IGraphTransformer::TStatus HasItemsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
6565
IGraphTransformer::TStatus ExtendWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
6666
IGraphTransformer::TStatus SelectOpWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
67-
IGraphTransformer::TStatus UnionAllPositionalWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx);
67+
IGraphTransformer::TStatus SelectOpPositionalWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx);
6868
template <bool IsStrict>
6969
IGraphTransformer::TStatus ListExtendWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx);
7070
IGraphTransformer::TStatus ListUnionAllWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);

‎yql/essentials/sql/v1/select.cpp

Lines changed: 18 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -2848,33 +2848,31 @@ class TSelectOp: public IRealSource {
28482848
}
28492849

28502850
TNodePtr Build(TContext& ctx) override {
2851-
TPtr res;
2851+
TString op;
2852+
28522853
if (Operator_ == "union") {
2853-
if (QuantifierAll_) {
2854-
if (ctx.EmitUnionMerge) {
2855-
res = ctx.PositionalUnionAll ? Y("UnionMergePositional") : Y("UnionMerge");
2856-
} else {
2857-
res = ctx.PositionalUnionAll ? Y("UnionAllPositional") : Y("UnionAll");
2858-
}
2859-
} else {
2860-
res = ctx.PositionalUnionAll ? Y("UnionPositional") : Y("Union");
2861-
}
2854+
op = "Union";
28622855
} else if (Operator_ == "intersect") {
2863-
if (QuantifierAll_) {
2864-
res = Y("IntersectAll");
2865-
} else {
2866-
res = Y("Intersect");
2867-
}
2856+
op = "Intersect";
28682857
} else if (Operator_ == "except") {
2869-
if (QuantifierAll_) {
2870-
res = Y("ExceptAll");
2871-
} else {
2872-
res = Y("Except");
2873-
}
2858+
op = "Except";
28742859
} else {
28752860
Y_ABORT("Invalid operator: %s", Operator_.c_str());
28762861
}
28772862

2863+
if (QuantifierAll_) {
2864+
if (Operator_ != "union" || !ctx.EmitUnionMerge) {
2865+
op += "All";
2866+
} else {
2867+
op += "Merge";
2868+
}
2869+
}
2870+
if (ctx.PositionalUnionAll) {
2871+
op += "Positional";
2872+
}
2873+
2874+
TPtr res = Y(op);
2875+
28782876
for (auto& s: Sources_) {
28792877
auto input = s->Build(ctx);
28802878
if (!input) {

‎yql/essentials/sql/v1/sql_ut_common.h

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1835,6 +1835,28 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) {
18351835
UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Intersect"]);
18361836
}
18371837

1838+
Y_UNIT_TEST(IntersectAllPositionalTest) {
1839+
NSQLTranslation::TTranslationSettings settings;
1840+
settings.LangVer = 202503;
1841+
NYql::TAstParseResult res = SqlToYqlWithSettings("PRAGMA PositionalUnionAll; SELECT key FROM plato.Input INTERSECT ALL select subkey FROM plato.Input;", settings);
1842+
UNIT_ASSERT(res.Root);
1843+
1844+
TWordCountHive elementStat = {{TString("IntersectAllPositional"), 0}};
1845+
VerifyProgram(res, elementStat, {});
1846+
UNIT_ASSERT_VALUES_EQUAL(1, elementStat["IntersectAllPositional"]);
1847+
}
1848+
1849+
Y_UNIT_TEST(IntersectDistinctPositionalTest) {
1850+
NSQLTranslation::TTranslationSettings settings;
1851+
settings.LangVer = 202503;
1852+
NYql::TAstParseResult res = SqlToYqlWithSettings("PRAGMA PositionalUnionAll; SELECT key FROM plato.Input INTERSECT DISTINCT select subkey FROM plato.Input;", settings);
1853+
UNIT_ASSERT(res.Root);
1854+
1855+
TWordCountHive elementStat = {{TString("IntersectPositional"), 0}};
1856+
VerifyProgram(res, elementStat, {});
1857+
UNIT_ASSERT_VALUES_EQUAL(1, elementStat["IntersectPositional"]);
1858+
}
1859+
18381860
// EXCEPT
18391861

18401862
Y_UNIT_TEST(ExceptAllTest) {
@@ -1870,6 +1892,28 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) {
18701892
UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Except"]);
18711893
}
18721894

1895+
Y_UNIT_TEST(ExceptAllPositionalTest) {
1896+
NSQLTranslation::TTranslationSettings settings;
1897+
settings.LangVer = 202503;
1898+
NYql::TAstParseResult res = SqlToYqlWithSettings("PRAGMA PositionalUnionAll; SELECT key FROM plato.Input EXCEPT ALL select subkey FROM plato.Input;", settings);
1899+
UNIT_ASSERT(res.Root);
1900+
1901+
TWordCountHive elementStat = {{TString("ExceptAllPositional"), 0}};
1902+
VerifyProgram(res, elementStat, {});
1903+
UNIT_ASSERT_VALUES_EQUAL(1, elementStat["ExceptAllPositional"]);
1904+
}
1905+
1906+
Y_UNIT_TEST(ExceptDistinctPositionalTest) {
1907+
NSQLTranslation::TTranslationSettings settings;
1908+
settings.LangVer = 202503;
1909+
NYql::TAstParseResult res = SqlToYqlWithSettings("PRAGMA PositionalUnionAll; SELECT key FROM plato.Input EXCEPT DISTINCT select subkey FROM plato.Input;", settings);
1910+
UNIT_ASSERT(res.Root);
1911+
1912+
TWordCountHive elementStat = {{TString("ExceptPositional"), 0}};
1913+
VerifyProgram(res, elementStat, {});
1914+
UNIT_ASSERT_VALUES_EQUAL(1, elementStat["ExceptPositional"]);
1915+
}
1916+
18731917

18741918
Y_UNIT_TEST(DeclareDecimalParameter) {
18751919
NYql::TAstParseResult res = SqlToYql("declare $value as Decimal(22,9); select $value as cnt;");

‎yql/essentials/tests/sql/minirun/part0/canondata/result.json

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -733,6 +733,20 @@
733733
"uri": "https://{canondata_backend}/995452/c22e7d9867fa56e2ee0c270fded39566aaa63a48/resource.tar.gz#test.test_flexible_types-group_by2-default.txt-Results_/results.txt"
734734
}
735735
],
736+
"test.test[intersect-intersect_all_positional-default.txt-Debug]": [
737+
{
738+
"checksum": "03739015c1f4bef2b2c0ec1d4b990a6b",
739+
"size": 2564,
740+
"uri": "https://{canondata_backend}/1942415/0f21a583e42ec6120ebf4ae7adaf85924f5930c1/resource.tar.gz#test.test_intersect-intersect_all_positional-default.txt-Debug_/opt.yql"
741+
}
742+
],
743+
"test.test[intersect-intersect_all_positional-default.txt-Results]": [
744+
{
745+
"checksum": "1737dd061908c24fa31d6dbd803743bc",
746+
"size": 4456,
747+
"uri": "https://{canondata_backend}/1942415/0f21a583e42ec6120ebf4ae7adaf85924f5930c1/resource.tar.gz#test.test_intersect-intersect_all_positional-default.txt-Results_/results.txt"
748+
}
749+
],
736750
"test.test[join-eq_over_join_bad_rotate-default.txt-Debug]": [
737751
{
738752
"checksum": "75bc473eee49f48848cfac6902f99607",

‎yql/essentials/tests/sql/minirun/part1/canondata/result.json

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -440,6 +440,34 @@
440440
"uri": "https://{canondata_backend}/1881367/453c70881991a95154014d718c871d0afdcbf15e/resource.tar.gz#test.test_except-except_all-default.txt-Results_/results.txt"
441441
}
442442
],
443+
"test.test[except-except_all_positional-default.txt-Debug]": [
444+
{
445+
"checksum": "54885f43cc523ba7a04a23168e45f8d6",
446+
"size": 2554,
447+
"uri": "https://{canondata_backend}/1942173/d09f4dbf08d7225751e84df7177a04a1bbd24999/resource.tar.gz#test.test_except-except_all_positional-default.txt-Debug_/opt.yql"
448+
}
449+
],
450+
"test.test[except-except_all_positional-default.txt-Results]": [
451+
{
452+
"checksum": "3a2887548ce5198b33b138f56f787a08",
453+
"size": 4897,
454+
"uri": "https://{canondata_backend}/1942173/d09f4dbf08d7225751e84df7177a04a1bbd24999/resource.tar.gz#test.test_except-except_all_positional-default.txt-Results_/results.txt"
455+
}
456+
],
457+
"test.test[except-except_distinct_positional-default.txt-Debug]": [
458+
{
459+
"checksum": "3de3f71338fe7d59297e034965574df3",
460+
"size": 2506,
461+
"uri": "https://{canondata_backend}/1942173/d09f4dbf08d7225751e84df7177a04a1bbd24999/resource.tar.gz#test.test_except-except_distinct_positional-default.txt-Debug_/opt.yql"
462+
}
463+
],
464+
"test.test[except-except_distinct_positional-default.txt-Results]": [
465+
{
466+
"checksum": "24f2ee28eff5c4c478d222cff5b11239",
467+
"size": 4189,
468+
"uri": "https://{canondata_backend}/1942173/d09f4dbf08d7225751e84df7177a04a1bbd24999/resource.tar.gz#test.test_except-except_distinct_positional-default.txt-Results_/results.txt"
469+
}
470+
],
443471
"test.test[expr-as_table_emptylist2-default.txt-Debug]": [
444472
{
445473
"checksum": "6424bb71484d775e2e9f4abcc3b852e6",

0 commit comments

Comments
 (0)