Skip to content

Commit cd8bd1f

Browse files
zverevgenyzverevgeny
authored
Integrate ilike pushdown (#20598)
2 parents 243a4e6 + 450e388 commit cd8bd1f

File tree

78 files changed

+660
-371
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

78 files changed

+660
-371
lines changed

‎ydb/core/formats/arrow/program/kernel_logic.h

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,54 @@ class TLogicMatchString: public IKernelLogic {
148148
}
149149
};
150150

151+
class TLogicMatchAsciiEqualsIgnoreCase: public TLogicMatchString {
152+
private:
153+
static TString GetClassNameStatic() {
154+
return "String._yql_AsciiEqualsIgnoreCase";
155+
}
156+
public:
157+
TLogicMatchAsciiEqualsIgnoreCase() : TLogicMatchString(TIndexCheckOperation::EOperation::Contains, false, false) {
158+
159+
}
160+
static const inline auto Registrator = TFactory::TRegistrator<TLogicMatchAsciiEqualsIgnoreCase>(GetClassNameStatic());
161+
};
162+
163+
class TLogicMatchAsciiContainsIgnoreCase: public TLogicMatchString {
164+
private:
165+
static TString GetClassNameStatic() {
166+
return "String._yql_AsciiContainsIgnoreCase";
167+
}
168+
public:
169+
TLogicMatchAsciiContainsIgnoreCase() : TLogicMatchString(TIndexCheckOperation::EOperation::Contains, false, false) {
170+
171+
}
172+
static const inline auto Registrator = TFactory::TRegistrator<TLogicMatchAsciiContainsIgnoreCase>(GetClassNameStatic());
173+
};
174+
175+
class TLogicMatchAsciiStartsWithIgnoreCase: public TLogicMatchString {
176+
private:
177+
static TString GetClassNameStatic() {
178+
return "String._yql_AsciiStartsWithIgnoreCase";
179+
}
180+
public:
181+
TLogicMatchAsciiStartsWithIgnoreCase() : TLogicMatchString(TIndexCheckOperation::EOperation::StartsWith, false, false) {
182+
183+
}
184+
static const inline auto Registrator = TFactory::TRegistrator<TLogicMatchAsciiStartsWithIgnoreCase>(GetClassNameStatic());
185+
};
186+
187+
class TLogicMatchAsciiEndsWithIgnoreCase: public TLogicMatchString {
188+
private:
189+
static TString GetClassNameStatic() {
190+
return "String._yql_AsciiEndsWithIgnoreCase";
191+
}
192+
public:
193+
TLogicMatchAsciiEndsWithIgnoreCase() : TLogicMatchString(TIndexCheckOperation::EOperation::EndsWith, false, false) {
194+
195+
}
196+
static const inline auto Registrator = TFactory::TRegistrator<TLogicMatchAsciiEndsWithIgnoreCase>(GetClassNameStatic());
197+
};
198+
151199
class TLogicEquals: public IKernelLogic {
152200
private:
153201
using TBase = IKernelLogic;

‎ydb/core/formats/arrow/ssa_runtime_version.h

Lines changed: 0 additions & 35 deletions
This file was deleted.

‎ydb/core/kqp/compile_service/kqp_compile_actor.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -654,6 +654,8 @@ void ApplyServiceConfig(TKikimrConfiguration& kqpConfig, const TTableServiceConf
654654
kqpConfig.EnableSnapshotIsolationRW = serviceConfig.GetEnableSnapshotIsolationRW();
655655
kqpConfig.EnableSpilling = serviceConfig.GetEnableQueryServiceSpilling();
656656
kqpConfig.EnableSpillingInHashJoinShuffleConnections = serviceConfig.GetEnableSpillingInHashJoinShuffleConnections();
657+
kqpConfig.EnableOlapScalarApply = serviceConfig.GetEnableOlapScalarApply();
658+
kqpConfig.EnableOlapSubstringPushdown = serviceConfig.GetEnableOlapSubstringPushdown();
657659

658660
if (const auto limit = serviceConfig.GetResourceManager().GetMkqlHeavyProgramMemoryLimit()) {
659661
kqpConfig._KqpYqlCombinerMemoryLimit = std::max(1_GB, limit - (limit >> 2U));

‎ydb/core/kqp/compile_service/kqp_compile_service.cpp

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -319,6 +319,8 @@ class TKqpCompileService : public TActorBootstrapped<TKqpCompileService> {
319319
bool enableSnapshotIsolationRW = TableServiceConfig.GetEnableSnapshotIsolationRW();
320320

321321
bool enableSpillingInHashJoinShuffleConnections = TableServiceConfig.GetEnableSpillingInHashJoinShuffleConnections();
322+
bool enableOlapScalarApply = TableServiceConfig.GetEnableOlapScalarApply();
323+
bool enableOlapSubstringPushdown = TableServiceConfig.GetEnableOlapSubstringPushdown();
322324

323325
TableServiceConfig.Swap(event.MutableConfig()->MutableTableServiceConfig());
324326
LOG_INFO(*TlsActivationContext, NKikimrServices::KQP_COMPILE_SERVICE, "Updated config");
@@ -353,7 +355,10 @@ class TKqpCompileService : public TActorBootstrapped<TKqpCompileService> {
353355
TableServiceConfig.GetEnableSnapshotIsolationRW() != enableSnapshotIsolationRW ||
354356
TableServiceConfig.GetEnableQueryServiceSpilling() != enableSpilling ||
355357
TableServiceConfig.GetDefaultEnableShuffleElimination() != defaultEnableShuffleElimination ||
356-
TableServiceConfig.GetEnableSpillingInHashJoinShuffleConnections() != enableSpillingInHashJoinShuffleConnections)
358+
TableServiceConfig.GetEnableSpillingInHashJoinShuffleConnections() != enableSpillingInHashJoinShuffleConnections ||
359+
TableServiceConfig.GetEnableOlapScalarApply() != enableOlapScalarApply ||
360+
TableServiceConfig.GetEnableOlapSubstringPushdown() != enableOlapSubstringPushdown
361+
)
357362
{
358363

359364
QueryCache->Clear();

‎ydb/core/kqp/expr_nodes/kqp_expr_nodes.json

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -682,15 +682,23 @@
682682
{"Index": 2, "Name": "ReturningType", "Type": "TExprBase"}
683683
]
684684
},
685+
{
686+
"Name": "TKqpOlapApplyColumnArg",
687+
"Base": "TExprBase",
688+
"Match": {"Type": "Callable", "Name": "KqpOlapApplyColumnArg"},
689+
"Children": [
690+
{"Index": 0, "Name": "TableRowType", "Type": "TExprBase"},
691+
{"Index": 1, "Name": "ColumnName", "Type": "TCoAtom"}
692+
]
693+
},
685694
{
686695
"Name": "TKqpOlapApply",
687696
"Base": "TExprBase",
688697
"Match": {"Type": "Callable", "Name": "KqpOlapApply"},
689698
"Children": [
690-
{"Index": 0, "Name": "Type", "Type": "TExprBase"},
691-
{"Index": 1, "Name": "Columns", "Type": "TCoAtomList"},
692-
{"Index": 2, "Name": "Parameters", "Type": "TExprList"},
693-
{"Index": 3, "Name": "Lambda", "Type": "TCoLambda"}
699+
{"Index": 0, "Name": "Lambda", "Type": "TCoLambda"},
700+
{"Index": 1, "Name": "Args", "Type": "TExprList"},
701+
{"Index": 2, "Name": "KernelName", "Type": "TCoAtom"}
694702
]
695703
},
696704
{

‎ydb/core/kqp/host/kqp_type_ann.cpp

Lines changed: 40 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1121,69 +1121,67 @@ TStatus AnnotateOlapFilter(const TExprNode::TPtr& node, TExprContext& ctx) {
11211121
return TStatus::Ok;
11221122
}
11231123

1124-
TStatus AnnotateOlapApply(const TExprNode::TPtr& node, TExprContext& ctx) {
1125-
if (!EnsureArgsCount(*node, 4U, ctx)) {
1124+
TStatus AnnotateOlapApplyColumnArg(const TExprNode::TPtr& node, TExprContext& ctx) {
1125+
if (!EnsureArgsCount(*node, 2U, ctx)) {
11261126
return TStatus::Error;
11271127
}
11281128

1129-
const auto type = node->Child(TKqpOlapApply::idx_Type);
1130-
if (!EnsureType(*type, ctx)) {
1129+
const auto& row = node->Head();
1130+
if (!EnsureType(row, ctx)) {
11311131
return TStatus::Error;
11321132
}
1133-
1134-
const auto argsType = type->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
1135-
if (!EnsureStructType(type->Pos(), *argsType, ctx)) {
1133+
const auto& rowType = row.GetTypeAnn()->Cast<TTypeExprType>()->GetType();
1134+
if (!EnsureStructType(row.Pos(), *rowType, ctx)) {
11361135
return TStatus::Error;
11371136
}
1137+
const auto& rowStructType = rowType->Cast<TStructExprType>();
11381138

1139-
const auto columns = node->Child(TKqpOlapApply::idx_Columns);
1140-
if (!EnsureTupleOfAtoms(*columns, ctx)) {
1139+
1140+
if (!EnsureAtom(node->Tail(), ctx)) {
11411141
return TStatus::Error;
11421142
}
1143-
1144-
const auto structType = argsType->Cast<TStructExprType>();
1145-
std::vector<const NYql::TTypeAnnotationNode*> argsTypes(columns->ChildrenSize());
1146-
1147-
for (auto i = 0U; i < argsTypes.size(); ++i) {
1148-
if (const auto argType = structType->FindItemType(columns->Child(i)->Content()))
1149-
argsTypes[i] = argType;
1150-
else {
1151-
ctx.AddError(TIssue(ctx.GetPosition(columns->Child(i)->Pos()),
1152-
TStringBuilder() << "Missed column: " << columns->Child(i)->Content()
1153-
));
1154-
return TStatus::Error;
1155-
}
1143+
const auto& columnName = node->Tail().Content();
1144+
if (const auto& columnType = rowStructType->FindItemType(columnName)) {
1145+
node->SetTypeAnn(columnType);
1146+
return TStatus::Ok;
1147+
} else {
1148+
ctx.AddError(TIssue(ctx.GetPosition(node->Tail().Pos()),
1149+
TStringBuilder() << "Missed column: " << columnName
1150+
));
1151+
return TStatus::Error;
11561152
}
1153+
}
11571154

1158-
TExprList parameters = TExprList(node->Child(TKqpOlapApply::idx_Parameters));
1159-
1160-
for(auto expr: parameters) {
1161-
if (!EnsureArgsCount(*expr.Ptr(), 2U, ctx)) {
1162-
return TStatus::Error;
1163-
}
1155+
TStatus AnnotateOlapApply(const TExprNode::TPtr& node, TExprContext& ctx) {
1156+
if (!EnsureArgsCount(*node, 3U, ctx)) {
1157+
return TStatus::Error;
1158+
}
11641159

1165-
TCoParameter param = TMaybeNode<TCoParameter>(expr.Ptr()).Cast();
1166-
const auto& paramType = expr.Ptr()->Child(TCoParameter::idx_Type);
1167-
if (!EnsureType(*paramType, ctx)) {
1168-
return TStatus::Error;
1169-
}
1160+
TExprList args = TExprList(node->Child(TKqpOlapApply::idx_Args));
1161+
std::vector<const NYql::TTypeAnnotationNode*> argTypes;
11701162

1171-
argsTypes.push_back(paramType->GetTypeAnn()->Cast<TTypeExprType>()->GetType());
1163+
for(const auto& arg: args) {
1164+
argTypes.push_back(arg.Ref().GetTypeAnn());
11721165
}
11731166

1174-
if (!EnsureLambda(node->Tail(), ctx)) {
1167+
auto& lambda = node->ChildRef(TKqpOlapApply::idx_Lambda);
1168+
if (!EnsureLambda(*lambda, ctx)) {
11751169
return TStatus::Error;
11761170
}
11771171

1178-
if (!UpdateLambdaAllArgumentsTypes(node->TailRef(), argsTypes, ctx)) {
1172+
if (!UpdateLambdaAllArgumentsTypes(lambda, argTypes, ctx)) {
11791173
return TStatus::Error;
11801174
}
11811175

1182-
if (!node->Tail().GetTypeAnn()) {
1176+
if (!lambda->GetTypeAnn()) {
11831177
return TStatus::Repeat;
11841178
}
11851179

1186-
node->SetTypeAnn(ctx.MakeType<TUnitExprType>());
1180+
if (!EnsureAtom(*node->Child(TKqpOlapApply::idx_KernelName), ctx)) {
1181+
return TStatus::Error;
1182+
}
1183+
1184+
node->SetTypeAnn(lambda->GetTypeAnn());
11871185
return TStatus::Ok;
11881186
}
11891187

@@ -1994,6 +1992,10 @@ TAutoPtr<IGraphTransformer> CreateKqpTypeAnnotationTransformer(const TString& cl
19941992
return AnnotateOlapFilter(input, ctx);
19951993
}
19961994

1995+
if (TKqpOlapApplyColumnArg::Match(input.Get())) {
1996+
return AnnotateOlapApplyColumnArg(input, ctx);
1997+
}
1998+
19971999
if (TKqpOlapApply::Match(input.Get())) {
19982000
return AnnotateOlapApply(input, ctx);
19992001
}

‎ydb/core/kqp/opt/physical/kqp_opt_phy_olap_agg.cpp

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@
22

33
#include <ydb/core/kqp/common/kqp_yql.h>
44

5-
#include <ydb/core/formats/arrow/ssa_runtime_version.h>
6-
75
#include <yql/essentials/core/yql_opt_utils.h>
86
#include <ydb/library/actors/core/log.h>
97

0 commit comments

Comments
 (0)