Skip to content

Commit 5898999

Browse files
vitstnKamil Khamitov
vitstn
authored and
Kamil Khamitov
committed
Optimizers by CheckMissingWorld flag
commit_hash:0450825a09ea7c8bedf6ed697c198eacef74da6c
1 parent 1c73927 commit 5898999

File tree

8 files changed

+245
-84
lines changed

8 files changed

+245
-84
lines changed

‎yql/essentials/core/common_opt/yql_co_simple1.cpp

Lines changed: 129 additions & 60 deletions
Large diffs are not rendered by default.

‎yql/essentials/core/common_opt/yql_co_simple2.cpp

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -809,13 +809,15 @@ TExprNode::TPtr CheckIfWorldWithSame(const TExprNode::TPtr& node, TExprContext&
809809
return node;
810810
}
811811

812-
TExprNode::TPtr CheckIfWithSame(const TExprNode::TPtr& node, TExprContext& ctx) {
812+
TExprNode::TPtr CheckIfWithSame(const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
813813
if (node->Child(node->ChildrenSize() - 1U) == node->Child(node->ChildrenSize() - 2U)) {
814814
YQL_CLOG(DEBUG, Core) << node->Content() << " with identical branches.";
815815
auto children = node->ChildrenList();
816816
children[children.size() - 3U] = std::move(children.back());
817817
children.resize(children.size() -2U);
818-
return 1U == children.size() ? children.front() : ctx.ChangeChildren(*node, std::move(children));
818+
auto res = 1U == children.size() ? children.front() : ctx.ChangeChildren(*node, std::move(children));
819+
res = KeepWorld(res, *node, ctx, *optCtx.Types);
820+
return res;
819821
}
820822

821823
if (const auto width = node->ChildrenSize() >> 1U; width > 1U) {
@@ -850,7 +852,9 @@ TExprNode::TPtr CheckIfWithSame(const TExprNode::TPtr& node, TExprContext& ctx)
850852
} else
851853
prev = ctx.NewCallable(node->Pos(), "Or", {std::move(prev), std::move(next)});
852854
children.erase(children.cbegin() + i + 1U, children.cbegin() + i + 3U);
853-
return ctx.ChangeChildren(*node, std::move(children));
855+
auto res = ctx.ChangeChildren(*node, std::move(children));
856+
res = KeepWorld(res, *node, ctx, *optCtx.Types);
857+
return res;
854858
}
855859
}
856860
}
@@ -860,10 +864,12 @@ TExprNode::TPtr CheckIfWithSame(const TExprNode::TPtr& node, TExprContext& ctx)
860864
}
861865

862866
template <bool Equal, bool Aggr>
863-
TExprNode::TPtr CheckCompareSame(const TExprNode::TPtr& node, TExprContext& ctx) {
867+
TExprNode::TPtr CheckCompareSame(const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
864868
if (&node->Head() == &node->Tail() && (Aggr || HasTotalOrder(*node->Head().GetTypeAnn()))) {
865869
YQL_CLOG(DEBUG, Core) << (Equal ? "Equal" : "Unequal") << " '" << node->Content() << "' with same args";
866-
return MakeBool<Equal>(node->Pos(), ctx);
870+
auto res = MakeBool<Equal>(node->Pos(), ctx);
871+
res = KeepWorld(res, *node, ctx, *optCtx.Types);
872+
return res;
867873
}
868874

869875
return node;
@@ -896,7 +902,7 @@ void RegisterCoSimpleCallables2(TCallableOptimizerMap& map) {
896902

897903
map[IfName] = std::bind(&CheckIfWorldWithSame, _1, _2);
898904

899-
map["If"] = std::bind(&CheckIfWithSame, _1, _2);
905+
map["If"] = &CheckIfWithSame;
900906

901907
map["Aggregate"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext&) {
902908
if (auto deduplicated = DeduplicateAggregateSameTraits(node, ctx); deduplicated != node) {
@@ -921,13 +927,13 @@ void RegisterCoSimpleCallables2(TCallableOptimizerMap& map) {
921927

922928
map["AggrMin"] = map["AggrMax"] = map["Coalesce"] = std::bind(&DropAggrOverSame, _1);
923929

924-
map["StartsWith"] = map["EndsWith"] = map["StringContains"] = std::bind(&CheckCompareSame<true, false>, _1, _2);
930+
map["StartsWith"] = map["EndsWith"] = map["StringContains"] = &CheckCompareSame<true, false>;
925931

926-
map["=="] = map["<="] = map[">="] = std::bind(&CheckCompareSame<true, false>, _1, _2);
927-
map["!="] = map["<"] = map[">"] = std::bind(&CheckCompareSame<false, false>, _1, _2);
932+
map["=="] = map["<="] = map[">="] = &CheckCompareSame<true, false>;
933+
map["!="] = map["<"] = map[">"] = &CheckCompareSame<false, false>;
928934

929-
map["AggrEquals"] = map["AggrLessOrEqual"] = map["AggrGreaterOrEqual"] = std::bind(&CheckCompareSame<true, true>, _1, _2);
930-
map["AggrNotEquals"] = map["AggrLess"] = map["AggrGreater"] = std::bind(&CheckCompareSame<false, true>, _1, _2);
935+
map["AggrEquals"] = map["AggrLessOrEqual"] = map["AggrGreaterOrEqual"] = &CheckCompareSame<true, true>;
936+
map["AggrNotEquals"] = map["AggrLess"] = map["AggrGreater"] = &CheckCompareSame<false, true>;
931937

932938
map["IfPresent"] = std::bind(&IfPresentSubsetFields, _1, _2, _3);
933939

‎yql/essentials/core/common_opt/yql_co_transformer.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -170,8 +170,8 @@ IGraphTransformer::TStatus TCommonOptTransformer::DoTransform(
170170
result = defaultOpt(node, ctx, optCtx);
171171
}
172172

173-
if (checkMissingWorld && result && result != node && !node->GetTypeAnn()->ReturnsWorld()) {
174-
if (KeepWorld(result, *node, ctx, *optCtx.Types) != result) {
173+
if (checkMissingWorld && result && result != node && !node->GetTypeAnn()->ReturnsWorld() && !node->IsCallable(RightName)) {
174+
if (HasMissingWorlds(result, *node, *optCtx.Types)) {
175175
throw yexception() << "Missing world over " << result->Dump();
176176
}
177177
}

‎yql/essentials/core/expr_nodes/yql_expr_nodes.json

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,15 @@
144144
{"Index": 1, "Name": "Input", "Type": "TExprBase"}
145145
]
146146
},
147+
{
148+
"Name": "TCoWithWorld",
149+
"Base": "TCallable",
150+
"Match": {"Type": "Callable", "Name": "WithWorld"},
151+
"Children": [
152+
{"Index": 0, "Name": "Input", "Type": "TExprBase"},
153+
{"Index": 1, "Name": "World", "Type": "TExprBase"}
154+
]
155+
},
147156
{
148157
"Name": "TCoTypeOf",
149158
"Base": "TCallable",

‎yql/essentials/core/peephole_opt/yql_opt_peephole_physical.cpp

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8754,7 +8754,6 @@ struct TPeepHoleRules {
87548754
{"GraceSelfJoinCore", &OptimizeGraceSelfJoinCore},
87558755
{"CommonJoinCore", &OptimizeCommonJoinCore},
87568756
{"BuildTablePath", &DoBuildTablePath},
8757-
{"Exists", &OptimizeExists},
87588757
{"SqueezeToDict", &OptimizeSqueezeToDict},
87598758
{"NarrowFlatMap", &OptimizeNarrowFlatMap},
87608759
{"NarrowMultiMap", &OptimizeWideMaps},
@@ -8774,7 +8773,9 @@ struct TPeepHoleRules {
87748773
{"Coalesce", &OptimizeCoalesce},
87758774
};
87768775

8777-
const TExtPeepHoleOptimizerMap FinalStageExtRules = {};
8776+
const TExtPeepHoleOptimizerMap FinalStageExtRules = {
8777+
{"Exists", &OptimizeExists},
8778+
};
87788779

87798780
const TExtPeepHoleOptimizerMap FinalStageNonDetRules = {
87808781
{"Random", &Random0Arg<double>},

‎yql/essentials/core/yql_opt_utils.cpp

Lines changed: 37 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1545,35 +1545,45 @@ template TExprNode::TPtr OptimizeIfPresent<true, true>(const TExprNode::TPtr& no
15451545
template TExprNode::TPtr OptimizeIfPresent<false, true>(const TExprNode::TPtr& node, TExprContext& ctx);
15461546
template TExprNode::TPtr OptimizeIfPresent<false, false>(const TExprNode::TPtr& node, TExprContext& ctx);
15471547

1548-
TExprNode::TPtr OptimizeExists(const TExprNode::TPtr& node, TExprContext& ctx) {
1548+
TExprNode::TPtr OptimizeExists(const TExprNode::TPtr& node, TExprContext& ctx, TTypeAnnotationContext& typeCtx) {
15491549
if (HasError(node->Head().GetTypeAnn(), ctx)) {
15501550
return TExprNode::TPtr();
15511551
}
15521552

15531553
if (node->Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Void) {
15541554
YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content();
1555-
return MakeBool<false>(node->Pos(), ctx);
1555+
auto res = MakeBool<false>(node->Pos(), ctx);
1556+
res = KeepWorld(res, *node, ctx, typeCtx);
1557+
return res;
15561558
}
15571559

15581560
if (node->Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Null) {
15591561
YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content();
1560-
return MakeBool<false>(node->Pos(), ctx);
1562+
auto res = MakeBool<false>(node->Pos(), ctx);
1563+
res = KeepWorld(res, *node, ctx, typeCtx);
1564+
return res;
15611565
}
15621566

15631567
if (node->Head().IsCallable({"Just", "PgConst"})) {
15641568
YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content();
1565-
return MakeBool<true>(node->Pos(), ctx);
1569+
auto res = MakeBool<true>(node->Pos(), ctx);
1570+
res = KeepWorld(res, *node, ctx, typeCtx);
1571+
return res;
15661572
}
15671573

15681574
if (node->Head().IsCallable({"Nothing","EmptyFrom"})) {
15691575
YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content();
1570-
return MakeBool<false>(node->Pos(), ctx);
1576+
auto res = MakeBool<false>(node->Pos(), ctx);
1577+
res = KeepWorld(res, *node, ctx, typeCtx);
1578+
return res;
15711579
}
15721580

15731581
if (node->Head().GetTypeAnn()->GetKind() != ETypeAnnotationKind::Optional &&
15741582
node->Head().GetTypeAnn()->GetKind() != ETypeAnnotationKind::Pg) {
15751583
YQL_CLOG(DEBUG, Core) << node->Content() << " over non-optional";
1576-
return MakeBool<true>(node->Pos(), ctx);
1584+
auto res = MakeBool<true>(node->Pos(), ctx);
1585+
res = KeepWorld(res, *node, ctx, typeCtx);
1586+
return res;
15771587
}
15781588

15791589
if (const auto& input = node->Head(); IsTransparentIfPresent(input)) {
@@ -2334,7 +2344,27 @@ bool CheckSupportedTypes(
23342344
return true;
23352345
}
23362346

2337-
TExprNode::TPtr KeepWorld(TExprNode::TPtr node, const TExprNode& src, TExprContext& ctx, TTypeAnnotationContext& types) {
2347+
bool HasMissingWorlds(const TExprNode::TPtr& node, const TExprNode& src, const TTypeAnnotationContext& types) {
2348+
const bool optEnabled = IsOptimizerEnabled<KeepWorldOptName>(types) && !IsOptimizerDisabled<KeepWorldOptName>(types);
2349+
if (!optEnabled) {
2350+
return false;
2351+
}
2352+
2353+
const auto& worldLinks = src.GetWorldLinks();
2354+
if (!worldLinks) {
2355+
return false;
2356+
}
2357+
2358+
for (const auto& link : *worldLinks) {
2359+
if (!IsDepended(*node, *link)) {
2360+
return true;
2361+
}
2362+
}
2363+
2364+
return false;
2365+
}
2366+
2367+
TExprNode::TPtr KeepWorld(TExprNode::TPtr node, const TExprNode& src, TExprContext& ctx, const TTypeAnnotationContext& types) {
23382368
const bool optEnabled = IsOptimizerEnabled<KeepWorldOptName>(types) && !IsOptimizerDisabled<KeepWorldOptName>(types);
23392369
if (!optEnabled) {
23402370
return node;

‎yql/essentials/core/yql_opt_utils.h

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ TExprNode::TPtr BuildKeySelector(TPositionHandle pos, const TStructExprType& row
119119

120120
template <bool Cannonize, bool EnableNewOptimizers = true>
121121
TExprNode::TPtr OptimizeIfPresent(const TExprNode::TPtr& node, TExprContext& ctx);
122-
TExprNode::TPtr OptimizeExists(const TExprNode::TPtr& node, TExprContext& ctx);
122+
TExprNode::TPtr OptimizeExists(const TExprNode::TPtr& node, TExprContext& ctx, TTypeAnnotationContext& typeCtx);
123123

124124
bool WarnUnroderedSubquery(const TExprNode& unourderedSubquery, TExprContext& ctx);
125125

@@ -151,7 +151,8 @@ bool HasDependsOn(const TExprNode::TPtr& node, const TExprNode::TPtr& arg);
151151
TExprNode::TPtr KeepSortedConstraint(TExprNode::TPtr node, const TSortedConstraintNode* sorted, const TTypeAnnotationNode* rowType, TExprContext& ctx);
152152
TExprNode::TPtr MakeSortByConstraint(TExprNode::TPtr node, const TSortedConstraintNode* sorted, const TTypeAnnotationNode* rowType, TExprContext& ctx);
153153
TExprNode::TPtr KeepConstraints(TExprNode::TPtr node, const TExprNode& src, TExprContext& ctx);
154-
TExprNode::TPtr KeepWorld(TExprNode::TPtr node, const TExprNode& src, TExprContext& ctx, TTypeAnnotationContext& types);
154+
bool HasMissingWorlds(const TExprNode::TPtr& node, const TExprNode& src, const TTypeAnnotationContext& types);
155+
TExprNode::TPtr KeepWorld(TExprNode::TPtr node, const TExprNode& src, TExprContext& ctx, const TTypeAnnotationContext& types);
155156

156157
void OptimizeSubsetFieldsForNodeWithMultiUsage(const TExprNode::TPtr& node, const TParentsMap& parentsMap,
157158
TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx,

‎yt/yql/providers/yt/provider/yql_yt_logical_optimize.cpp

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ class TYtLogicalOptProposalTransformer : public TOptimizeTransformerBase {
3939
, State_(state)
4040
{
4141
#define HNDL(name) "LogicalOptimizer-"#name, Hndl(&TYtLogicalOptProposalTransformer::name)
42+
AddHandler(0, &TCoWithWorld::Match, HNDL(WithWorld));
4243
AddHandler(0, &TYtMap::Match, HNDL(DirectRow));
4344
AddHandler(0, Names({TYtReduce::CallableName(), TYtMapReduce::CallableName()}), HNDL(IsKeySwitch));
4445
AddHandler(0, &TCoLeft::Match, HNDL(TrimReadWorld));
@@ -570,6 +571,50 @@ class TYtLogicalOptProposalTransformer : public TOptimizeTransformerBase {
570571
.Done();
571572
}
572573

574+
TMaybeNode<TExprBase> WithWorld(TExprBase node, TExprContext& ctx) const {
575+
auto maybeRead = node.Cast<TCoWithWorld>().Input().Maybe<TCoRight>().Input().Maybe<TYtReadTable>();
576+
if (maybeRead) {
577+
auto read = maybeRead.Cast();
578+
TExprNode::TListType worlds(1, read.World().Ptr());
579+
worlds.push_back(node.Cast<TCoWithWorld>().World().Ptr());
580+
auto sync = ctx.NewCallable(node.Pos(), TCoSync::CallableName(), std::move(worlds));
581+
return Build<TCoRight>(ctx, node.Pos())
582+
.Input<TYtReadTable>()
583+
.InitFrom(read)
584+
.World(sync)
585+
.Build()
586+
.Done();
587+
}
588+
589+
auto maybeOut = node.Cast<TCoWithWorld>().Input().Maybe<TYtOutput>();
590+
if (maybeOut) {
591+
TVector<TYtSection> sections;
592+
sections.push_back(Build<TYtSection>(ctx, node.Pos())
593+
.Paths()
594+
.Add()
595+
.Table(maybeOut.Cast())
596+
.Columns<TCoVoid>().Build()
597+
.Ranges<TCoVoid>().Build()
598+
.Stat<TCoVoid>().Build()
599+
.Build()
600+
.Build()
601+
.Settings().Build()
602+
.Done());
603+
604+
return Build<TCoRight>(ctx, node.Pos())
605+
.Input<TYtReadTable>()
606+
.World(node.Cast<TCoWithWorld>().World())
607+
.DataSource(GetDataSource(maybeOut.Cast(), ctx))
608+
.Input()
609+
.Add(sections)
610+
.Build()
611+
.Build()
612+
.Done();
613+
}
614+
615+
return node;
616+
}
617+
573618
TMaybeNode<TExprBase> TrimReadWorld(TExprBase node, TExprContext& ctx) const {
574619
auto maybeRead = node.Cast<TCoLeft>().Input().Maybe<TYtReadTable>();
575620
if (!maybeRead) {

0 commit comments

Comments
 (0)