Skip to content

Commit df7e5a0

Browse files
author
Vadim Averin
authored
Add hint for compact Join (#10236)
1 parent 6e67eb5 commit df7e5a0

File tree

19 files changed

+177
-39
lines changed

19 files changed

+177
-39
lines changed

‎ydb/library/yql/core/type_ann/type_ann_join.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -900,6 +900,11 @@ namespace NTypeAnnImpl {
900900
else if (optionName == "join_algo") {
901901
// do nothing
902902
}
903+
else if (optionName == "compact") {
904+
if (!EnsureTupleSize(*child, 1, ctx.Expr)) {
905+
return IGraphTransformer::TStatus::Error;
906+
}
907+
}
903908
else {
904909
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() <<
905910
"Unknown option name: " << optionName));

‎ydb/library/yql/core/yql_join.cpp

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -332,6 +332,11 @@ namespace {
332332
else if (option.IsAtom("join_algo")) {
333333
//do nothing
334334
}
335+
else if (option.IsAtom("compact")) {
336+
if (!EnsureTupleSize(*child, 1, ctx)) {
337+
return IGraphTransformer::TStatus::Error;
338+
}
339+
}
335340
else {
336341
ctx.AddError(TIssue(ctx.GetPosition(option.Pos()), TStringBuilder() <<
337342
"Unknown option name: " << option.Content()));
@@ -774,6 +779,8 @@ IGraphTransformer::TStatus ValidateEquiJoinOptions(TPositionHandle positionHandl
774779
// do nothing
775780
} else if (optionName == "join_algo") {
776781
// do nothing
782+
} else if (optionName == "compact") {
783+
options.Compact = true;
777784
} else {
778785
ctx.AddError(TIssue(position, TStringBuilder() <<
779786
"Unknown option name: " << optionName));
@@ -1345,10 +1352,14 @@ TEquiJoinLinkSettings GetEquiJoinLinkSettings(const TExprNode& linkSettings) {
13451352

13461353
result.ForceSortedMerge = HasSetting(linkSettings, "forceSortedMerge");
13471354

1348-
if(HasSetting(linkSettings, "forceStreamLookup")) {
1355+
if (HasSetting(linkSettings, "forceStreamLookup")) {
13491356
result.JoinAlgo = EJoinAlgoType::StreamLookupJoin;
13501357
}
13511358

1359+
if (HasSetting(linkSettings, "compact")) {
1360+
result.Compact = true;
1361+
}
1362+
13521363
return result;
13531364
}
13541365

@@ -1382,6 +1393,10 @@ TExprNode::TPtr BuildEquiJoinLinkSettings(const TEquiJoinLinkSettings& linkSetti
13821393
settings.push_back(builder("right"));
13831394
}
13841395

1396+
if (linkSettings.Compact) {
1397+
settings.push_back(ctx.NewList(linkSettings.Pos, { ctx.NewAtom(linkSettings.Pos, "compact", TNodeFlags::Default) }));
1398+
}
1399+
13851400
return ctx.NewList(linkSettings.Pos, std::move(settings));
13861401
}
13871402

‎ydb/library/yql/core/yql_join.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ struct TJoinOptions {
6060

6161
bool Flatten = false;
6262
bool StrictKeys = false;
63+
bool Compact = false;
6364
};
6465

6566
IGraphTransformer::TStatus ValidateEquiJoinOptions(
@@ -146,6 +147,7 @@ struct TEquiJoinLinkSettings {
146147
EJoinAlgoType JoinAlgo = EJoinAlgoType::Undefined;
147148
// JOIN implementation may ignore this flags if SortedMerge strategy is not supported
148149
bool ForceSortedMerge = false;
150+
bool Compact = false;
149151
};
150152

151153
TEquiJoinLinkSettings GetEquiJoinLinkSettings(const TExprNode& linkSettings);

‎ydb/library/yql/providers/yt/provider/yql_yt_join_impl.cpp

Lines changed: 37 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -3308,47 +3308,49 @@ TStatus RewriteYtEquiJoinLeaf(TYtEquiJoin equiJoin, TYtJoinNodeOp& op, TYtJoinNo
33083308
bool useJoinReduceForSecond = false;
33093309
bool tryFirstAsPrimary = false;
33103310

3311-
if (allowPrimaryLeft != allowPrimaryRight) {
3312-
swapTables = allowPrimaryLeft;
3313-
auto primary = swapTables ? TChoice::Left : TChoice::Right;
3314-
useJoinReduce = !HasNonTrivialAny(linkSettings, mapSettings, primary);
3315-
} else if (allowPrimaryLeft) {
3316-
YQL_ENSURE(allowPrimaryRight);
3317-
// both tables can be chosen as primary
3318-
bool biggerHasUniqueKeys = mapSettings.RightSize > mapSettings.LeftSize ?
3319-
mapSettings.RightUnique : mapSettings.LeftUnique;
3320-
3321-
if (biggerHasUniqueKeys) {
3322-
// it is safe to use smaller table as primary
3323-
swapTables = mapSettings.RightSize > mapSettings.LeftSize;
3324-
} else if (mergeUseSmallAsPrimary) {
3325-
// explicit setting
3326-
if (*mergeUseSmallAsPrimary) {
3327-
// use smaller table as primary
3311+
if (!linkSettings.Compact) {
3312+
if (allowPrimaryLeft != allowPrimaryRight) {
3313+
swapTables = allowPrimaryLeft;
3314+
auto primary = swapTables ? TChoice::Left : TChoice::Right;
3315+
useJoinReduce = !HasNonTrivialAny(linkSettings, mapSettings, primary);
3316+
} else if (allowPrimaryLeft) {
3317+
YQL_ENSURE(allowPrimaryRight);
3318+
// both tables can be chosen as primary
3319+
bool biggerHasUniqueKeys = mapSettings.RightSize > mapSettings.LeftSize ?
3320+
mapSettings.RightUnique : mapSettings.LeftUnique;
3321+
3322+
if (biggerHasUniqueKeys) {
3323+
// it is safe to use smaller table as primary
33283324
swapTables = mapSettings.RightSize > mapSettings.LeftSize;
3325+
} else if (mergeUseSmallAsPrimary) {
3326+
// explicit setting
3327+
if (*mergeUseSmallAsPrimary) {
3328+
// use smaller table as primary
3329+
swapTables = mapSettings.RightSize > mapSettings.LeftSize;
3330+
} else {
3331+
// use bigger table as primary
3332+
swapTables = mapSettings.LeftSize > mapSettings.RightSize;
3333+
}
33293334
} else {
3330-
// use bigger table as primary
3335+
// make bigger table last one, and try first (smaller) as primary
33313336
swapTables = mapSettings.LeftSize > mapSettings.RightSize;
3337+
tryFirstAsPrimary = true;
33323338
}
3333-
} else {
3334-
// make bigger table last one, and try first (smaller) as primary
3335-
swapTables = mapSettings.LeftSize > mapSettings.RightSize;
3336-
tryFirstAsPrimary = true;
3337-
}
33383339

3339-
auto primary = swapTables ? TChoice::Left : TChoice::Right;
3340-
if (tryFirstAsPrimary) {
3341-
useJoinReduceForSecond = !HasNonTrivialAny(linkSettings, mapSettings, primary);
3342-
useJoinReduce = !HasNonTrivialAny(linkSettings, mapSettings, Invert(primary));
3343-
} else {
3344-
useJoinReduce = !HasNonTrivialAny(linkSettings, mapSettings, primary);
3345-
}
3346-
} else {
3347-
// try to move non-fat table to the left, otherwise keep them as is
3348-
if (mapSettings.LeftUnique != mapSettings.RightUnique) {
3349-
swapTables = mapSettings.RightUnique;
3340+
auto primary = swapTables ? TChoice::Left : TChoice::Right;
3341+
if (tryFirstAsPrimary) {
3342+
useJoinReduceForSecond = !HasNonTrivialAny(linkSettings, mapSettings, primary);
3343+
useJoinReduce = !HasNonTrivialAny(linkSettings, mapSettings, Invert(primary));
3344+
} else {
3345+
useJoinReduce = !HasNonTrivialAny(linkSettings, mapSettings, primary);
3346+
}
33503347
} else {
3351-
swapTables = mapSettings.LeftSize > mapSettings.RightSize;
3348+
// try to move non-fat table to the left, otherwise keep them as is
3349+
if (mapSettings.LeftUnique != mapSettings.RightUnique) {
3350+
swapTables = mapSettings.RightUnique;
3351+
} else {
3352+
swapTables = mapSettings.LeftSize > mapSettings.RightSize;
3353+
}
33523354
}
33533355
}
33543356

‎ydb/library/yql/sql/v1/join.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -515,6 +515,10 @@ class TEquiJoin: public TJoinBase {
515515
linkOptions = L(linkOptions, Q(Y(Q("right"), Q("any"))));
516516
}
517517

518+
if (descr.LinkSettings.Compact) {
519+
linkOptions = L(linkOptions, Q(Y(Q("compact"))));
520+
}
521+
518522
joinTree = Q(Y(
519523
Q(descr.Op),
520524
leftBranch,

‎ydb/library/yql/sql/v1/source.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,7 @@ namespace NSQLTranslationV1 {
172172
ForceGrace
173173
};
174174
EStrategy Strategy = EStrategy::Default;
175+
bool Compact = false;
175176
};
176177

177178
class IJoin: public ISource {

‎ydb/library/yql/sql/v1/sql_select.cpp

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,11 @@ bool CollectJoinLinkSettings(TPosition pos, TJoinLinkSettings& linkSettings, TCo
3434
newStrategy = TJoinLinkSettings::EStrategy::ForceMap;
3535
} else if (canonizedName == "grace") {
3636
newStrategy = TJoinLinkSettings::EStrategy::ForceGrace;
37+
} else if (canonizedName == "compact") {
38+
linkSettings.Compact = true;
39+
continue;
3740
} else {
38-
ctx.Warning(hint.Pos, TIssuesIds::YQL_UNUSED_HINT) << "Unsupported join strategy: " << hint.Name;
41+
ctx.Warning(hint.Pos, TIssuesIds::YQL_UNUSED_HINT) << "Unsupported join hint: " << hint.Name;
3942
}
4043

4144
if (TJoinLinkSettings::EStrategy::Default == linkSettings.Strategy) {

‎ydb/library/yql/sql/v1/sql_ut.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -571,7 +571,7 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) {
571571
Y_UNIT_TEST(WarnUnknownJoinStrategyHint) {
572572
NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input AS a JOIN /*+ xmerge() */ plato.Input AS b USING (key);");
573573
UNIT_ASSERT(res.Root);
574-
UNIT_ASSERT_STRINGS_EQUAL(res.Issues.ToString(), "<main>:1:41: Warning: Unsupported join strategy: xmerge, code: 4534\n");
574+
UNIT_ASSERT_STRINGS_EQUAL(res.Issues.ToString(), "<main>:1:41: Warning: Unsupported join hint: xmerge, code: 4534\n");
575575
}
576576

577577
Y_UNIT_TEST(ReverseLabels) {

‎ydb/library/yql/sql/v1/sql_ut_antlr4.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -571,7 +571,7 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) {
571571
Y_UNIT_TEST(WarnUnknownJoinStrategyHint) {
572572
NYql::TAstParseResult res = SqlToYql("SELECT * FROM plato.Input AS a JOIN /*+ xmerge() */ plato.Input AS b USING (key);");
573573
UNIT_ASSERT(res.Root);
574-
UNIT_ASSERT_STRINGS_EQUAL(res.Issues.ToString(), "<main>:1:41: Warning: Unsupported join strategy: xmerge, code: 4534\n");
574+
UNIT_ASSERT_STRINGS_EQUAL(res.Issues.ToString(), "<main>:1:41: Warning: Unsupported join hint: xmerge, code: 4534\n");
575575
}
576576

577577
Y_UNIT_TEST(ReverseLabels) {

‎ydb/library/yql/tests/sql/dq_file/part9/canondata/result.json

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1153,6 +1153,28 @@
11531153
}
11541154
],
11551155
"test.test[insert_monotonic-several2-default.txt-Results]": [],
1156+
"test.test[join-compact_join--Analyze]": [
1157+
{
1158+
"checksum": "7053ac9619fde0e66ec5651f1b9449e4",
1159+
"size": 7788,
1160+
"uri": "https://{canondata_backend}/1599023/7241ba5b766444adac2e1b71063e3efa9d1ac6af/resource.tar.gz#test.test_join-compact_join--Analyze_/plan.txt"
1161+
}
1162+
],
1163+
"test.test[join-compact_join--Debug]": [
1164+
{
1165+
"checksum": "8d5406ab6ae26cbcf9745ec54e5e6849",
1166+
"size": 3193,
1167+
"uri": "https://{canondata_backend}/1599023/7241ba5b766444adac2e1b71063e3efa9d1ac6af/resource.tar.gz#test.test_join-compact_join--Debug_/opt.yql_patched"
1168+
}
1169+
],
1170+
"test.test[join-compact_join--Plan]": [
1171+
{
1172+
"checksum": "7053ac9619fde0e66ec5651f1b9449e4",
1173+
"size": 7788,
1174+
"uri": "https://{canondata_backend}/1599023/7241ba5b766444adac2e1b71063e3efa9d1ac6af/resource.tar.gz#test.test_join-compact_join--Plan_/plan.txt"
1175+
}
1176+
],
1177+
"test.test[join-compact_join--Results]": [],
11561178
"test.test[join-inner_grouped--Analyze]": [
11571179
{
11581180
"checksum": "90bfcfc8052cd8a05de2934a9f39cfa0",

0 commit comments

Comments
 (0)