Skip to content

Commit d734760

Browse files
pashandor789pashandor789
authored
[CBO] Hints parser added (#9252)
1 parent 162c9f5 commit d734760

19 files changed

+528
-219
lines changed

‎ydb/core/kqp/opt/kqp_opt.h

Lines changed: 10 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ struct TKqpOptimizeContext : public TSimpleRefCount<TKqpOptimizeContext> {
1818
, QueryCtx(queryCtx)
1919
, Tables(tables)
2020
, UserRequestContext(userRequestContext)
21-
{
21+
{
2222
YQL_ENSURE(QueryCtx);
2323
YQL_ENSURE(Tables);
2424
}
@@ -31,9 +31,7 @@ struct TKqpOptimizeContext : public TSimpleRefCount<TKqpOptimizeContext> {
3131
int JoinsCount{};
3232
int EquiJoinsCount{};
3333
std::shared_ptr<NJson::TJsonValue> OverrideStatistics{};
34-
std::shared_ptr<NYql::TCardinalityHints> CardinalityHints{};
35-
std::shared_ptr<NYql::TJoinAlgoHints> JoinAlgoHints{};
36-
std::shared_ptr<NYql::TJoinOrderHints> JoinOrderHints{};
34+
std::shared_ptr<NYql::TOptimizerHints> Hints{};
3735

3836
std::shared_ptr<NJson::TJsonValue> GetOverrideStatistics() {
3937
if (Config->OptOverrideStatistics.Get()) {
@@ -49,37 +47,17 @@ struct TKqpOptimizeContext : public TSimpleRefCount<TKqpOptimizeContext> {
4947
}
5048
}
5149

52-
NYql::TCardinalityHints GetCardinalityHints() {
53-
if (Config->OptCardinalityHints.Get()) {
54-
if (!CardinalityHints) {
55-
CardinalityHints = std::make_shared<NYql::TCardinalityHints>(*Config->OptCardinalityHints.Get());
50+
NYql::TOptimizerHints GetOptimizerHints() {
51+
if (Config->OptimizerHints.Get()) {
52+
if (!Hints) {
53+
Hints = std::make_shared<NYql::TOptimizerHints>(
54+
NYql::TOptimizerHints::Parse(*Config->OptimizerHints.Get())
55+
);
5656
}
57-
return *CardinalityHints;
58-
} else {
59-
return NYql::TCardinalityHints();
60-
}
61-
}
62-
63-
NYql::TJoinAlgoHints GetJoinAlgoHints() {
64-
if (Config->OptJoinAlgoHints.Get()) {
65-
if (!JoinAlgoHints) {
66-
JoinAlgoHints = std::make_shared<NYql::TJoinAlgoHints>(*Config->OptJoinAlgoHints.Get());
67-
}
68-
return *JoinAlgoHints;
69-
} else {
70-
return NYql::TJoinAlgoHints();
57+
return *Hints;
7158
}
72-
}
7359

74-
NYql::TJoinOrderHints GetJoinOrderHints() {
75-
if (Config->OptJoinOrderHints.Get()) {
76-
if (!JoinOrderHints) {
77-
JoinOrderHints = std::make_shared<NYql::TJoinOrderHints>(*Config->OptJoinOrderHints.Get());
78-
}
79-
return *JoinOrderHints;
80-
} else {
81-
return NYql::TJoinOrderHints();
82-
}
60+
return NYql::TOptimizerHints();
8361
}
8462

8563
bool IsDataQuery() const {

‎ydb/core/kqp/opt/kqp_statistics_transformer.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ class TKqpStatisticsTransformer : public NYql::NDq::TDqStatisticsTransformerBase
3535
public:
3636
TKqpStatisticsTransformer(const TIntrusivePtr<TKqpOptimizeContext>& kqpCtx, TTypeAnnotationContext& typeCtx,
3737
const TKikimrConfiguration::TPtr& config, const TKqpProviderContext& pctx) :
38-
TDqStatisticsTransformerBase(&typeCtx, pctx, kqpCtx->GetCardinalityHints()),
38+
TDqStatisticsTransformerBase(&typeCtx, pctx, *kqpCtx->GetOptimizerHints().CardinalityHints),
3939
Config(config),
4040
KqpCtx(*kqpCtx) {}
4141

‎ydb/core/kqp/opt/logical/kqp_opt_log.cpp

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include <ydb/library/yql/dq/opt/dq_opt_join.h>
1212
#include <ydb/library/yql/dq/opt/dq_opt_log.h>
1313
#include <ydb/library/yql/dq/opt/dq_opt_hopping.h>
14+
#include <ydb/library/yql/utils/log/log.h>
1415
#include <ydb/library/yql/providers/common/transform/yql_optimize.h>
1516
#include <ydb/library/yql/providers/dq/common/yql_dq_settings.h>
1617

@@ -87,6 +88,17 @@ class TKqpLogicalOptTransformer : public TOptimizeTransformerBase {
8788
SetGlobal(4u);
8889
}
8990

91+
public:
92+
TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) override {
93+
auto status = TOptimizeTransformerBase::DoTransform(input, output, ctx);
94+
95+
for (const auto& hint: KqpCtx.GetOptimizerHints().GetUnappliedHintStrings()) {
96+
YQL_CLOG(WARN, ProviderYdb) << "Unapplied hint: " + hint;
97+
}
98+
99+
return status;
100+
}
101+
90102
protected:
91103
TMaybeNode<TExprBase> PushExtractedPredicateToReadTable(TExprBase node, TExprContext& ctx, const TGetParents& getParents) {
92104
TExprBase output = KqpPushExtractedPredicateToReadTable(node, ctx, KqpCtx, TypesCtx, *getParents());
@@ -155,11 +167,7 @@ class TKqpLogicalOptTransformer : public TOptimizeTransformerBase {
155167
rels.emplace_back(std::make_shared<TKqpRelOptimizerNode>(TString(label), stat, node));
156168
},
157169
KqpCtx.EquiJoinsCount,
158-
TOptimizerHints{
159-
.CardinalityHints = KqpCtx.GetCardinalityHints(),
160-
.JoinAlgoHints = KqpCtx.GetJoinAlgoHints(),
161-
.JoinOrderHints = KqpCtx.GetJoinOrderHints()
162-
}
170+
KqpCtx.GetOptimizerHints()
163171
);
164172
DumpAppliedRule("OptimizeEquiJoinWithCosts", node.Ptr(), output.Ptr(), ctx);
165173
return output;

‎ydb/core/kqp/opt/logical/kqp_opt_log.h

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,10 @@ namespace NKikimr::NKqp::NOpt {
77

88
struct TKqpOptimizeContext;
99

10-
TAutoPtr<NYql::IGraphTransformer> CreateKqpLogOptTransformer(TIntrusivePtr<TKqpOptimizeContext>& kqpCtx,
11-
NYql::TTypeAnnotationContext& typesCtx, const NYql::TKikimrConfiguration::TPtr& config);
10+
TAutoPtr<NYql::IGraphTransformer> CreateKqpLogOptTransformer(
11+
TIntrusivePtr<TKqpOptimizeContext>& kqpCtx,
12+
NYql::TTypeAnnotationContext& typesCtx,
13+
const NYql::TKikimrConfiguration::TPtr& config
14+
);
1215

1316
} // namespace NKikimr::NKqp::NOpt

‎ydb/core/kqp/provider/yql_kikimr_settings.cpp

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -82,9 +82,7 @@ TKikimrConfiguration::TKikimrConfiguration() {
8282
REGISTER_SETTING(*this, OptEnableOlapPushdown);
8383
REGISTER_SETTING(*this, OptEnableOlapProvideComputeSharding);
8484
REGISTER_SETTING(*this, OptOverrideStatistics);
85-
REGISTER_SETTING(*this, OptCardinalityHints);
86-
REGISTER_SETTING(*this, OptJoinAlgoHints);
87-
REGISTER_SETTING(*this, OptJoinOrderHints);
85+
REGISTER_SETTING(*this, OptimizerHints);
8886
REGISTER_SETTING(*this, OverridePlanner);
8987
REGISTER_SETTING(*this, UseGraceJoinCoreForMap);
9088

‎ydb/core/kqp/provider/yql_kikimr_settings.h

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,7 @@ struct TKikimrSettings {
5555
NCommon::TConfSetting<bool, false> UseGraceJoinCoreForMap;
5656

5757
NCommon::TConfSetting<TString, false> OptOverrideStatistics;
58-
NCommon::TConfSetting<TString, false> OptCardinalityHints;
59-
NCommon::TConfSetting<TString, false> OptJoinAlgoHints;
60-
NCommon::TConfSetting<TString, false> OptJoinOrderHints;
58+
NCommon::TConfSetting<TString, false> OptimizerHints;
6159

6260
/* Disable optimizer rules */
6361
NCommon::TConfSetting<bool, false> OptDisableTopSort;

‎ydb/core/kqp/ut/join/data/queries/join_order_hints_complex.sql

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,18 @@
11
PRAGMA TablePathPrefix='/Root';
22

3-
PRAGMA ydb.OptJoinOrderHints=
4-
'[
5-
[["R", "S"], ["T", "U"]]
6-
]';
3+
PRAGMA ydb.OptimizerHints =
4+
'
5+
Card(Unused # 10e8)
6+
JoinOrder( (Unused1 Unused2) (Unused3 Unused4) )
77
8-
PRAGMA ydb.OptCardinalityHints =
9-
'[
10-
{"labels":["R"], "op":"#", "value":10e8},
11-
{"labels":["T"], "op":"#", "value":1},
12-
{"labels":["R", "T"], "op":"#", "value":1},
13-
{"labels":["R", "S"], "op":"#", "value":10e8},
14-
{"labels":["T", "U"], "op":"#", "value":10e8},
15-
{"labels":["V"], "op":"#", "value":1}
16-
]';
8+
Card(R # 10e8)
9+
Card(T # 1)
10+
Card(R T # 1)
11+
Card(R S # 10e8)
12+
Card(T U # 10e8)
13+
Card(V # 1)
14+
JoinOrder( (R S) (T U) )
15+
';
1716

1817
SELECT * FROM
1918
R INNER JOIN S on R.id = S.id

‎ydb/core/kqp/ut/join/data/queries/join_order_hints_many_hint_trees.sql

Lines changed: 11 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,16 @@
11
PRAGMA TablePathPrefix='/Root';
22

3-
PRAGMA ydb.OptJoinOrderHints=
4-
'[
5-
["R", "S"],
6-
["T", "U"]
7-
]';
8-
PRAGMA ydb.OptCardinalityHints =
9-
'[
10-
{"labels":["R"], "op":"#", "value":10e8},
11-
{"labels":["T"], "op":"#", "value":1},
12-
{"labels":["R", "T"], "op":"#", "value":1},
13-
{"labels":["R", "S"], "op":"#", "value":10e8},
14-
{"labels":["T", "U"], "op":"#", "value":10e8},
15-
{"labels":["V"], "op":"#", "value":1}
16-
]';
3+
PRAGMA ydb.OptimizerHints =
4+
'
5+
Card(R # 10e8)
6+
Card(T # 1)
7+
Card(R T # 1)
8+
Card(R S # 10e8)
9+
Card(T U # 10e8)
10+
Card(V # 1)
11+
JoinOrder(T U)
12+
JoinOrder(R S)
13+
';
1714

1815
SELECT * FROM
1916
R INNER JOIN S on R.id = S.id

‎ydb/core/kqp/ut/join/data/queries/join_order_hints_simple.sql

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,14 @@
11
PRAGMA TablePathPrefix='/Root';
22

3-
PRAGMA ydb.OptCardinalityHints =
4-
'[
5-
{"labels":["R"], "op":"#", "value":10e8},
6-
{"labels":["T"], "op":"#", "value":1},
7-
{"labels":["S"], "op":"#", "value":10e8},
8-
{"labels":["R", "T"], "op":"#", "value":1},
9-
{"labels":["R", "S"], "op":"#", "value":10e8}
10-
]';
11-
PRAGMA ydb.OptJoinOrderHints=
12-
'[
13-
["T", ["R", "S"]]
14-
]';
3+
PRAGMA ydb.OptimizerHints =
4+
'
5+
Card(R # 10e8)
6+
Card(T # 1)
7+
Card(S # 10e8)
8+
Card(R T # 1)
9+
Card(R S # 10e8)
10+
JoinOrder(T (R S))
11+
';
1512

1613
SELECT * FROM
1714
R INNER JOIN S on R.id = S.id

‎ydb/core/kqp/ut/join/data/queries/test_join_hint.sql

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
1-
PRAGMA ydb.OptCardinalityHints = '[{"labels":["R"], "op":"#", "value":1}, {"labels":["R","S"], "op":"#", "value":10e6}]';
2-
PRAGMA ydb.OptJoinAlgoHints = '[{"labels":["R","S"], "algo":"GraceJoin"}]';
1+
PRAGMA ydb.OptimizerHints =
2+
'
3+
JoinAlgo(R S Grace)
4+
Card(R # 1)
5+
Card(R S # 10e6)
6+
';
37

48
SELECT *
59
FROM `/Root/R` as R

0 commit comments

Comments
 (0)