Pushdown ilike (#18540)

zverevgeny · zverevgeny · commit ccd1e9746d1e · 2025-07-07T13:41:08.000+03:00
diff --git a/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json b/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json
@@ -697,7 +697,8 @@
             "Match": {"Type": "Callable", "Name": "KqpOlapApply"},
             "Children": [
                 {"Index": 0, "Name": "Lambda", "Type": "TCoLambda"},
-                {"Index": 1, "Name": "Args", "Type": "TExprList"}
+                {"Index": 1, "Name": "Args", "Type": "TExprList"},
+                {"Index": 2, "Name": "KernelName", "Type": "TCoAtom"}
             ]
         },
         {
diff --git a/ydb/core/kqp/host/kqp_type_ann.cpp b/ydb/core/kqp/host/kqp_type_ann.cpp
@@ -1153,7 +1153,7 @@ TStatus AnnotateOlapApplyColumnArg(const TExprNode::TPtr& node, TExprContext& ct
 }
 
 TStatus AnnotateOlapApply(const TExprNode::TPtr& node, TExprContext& ctx) {
-    if (!EnsureArgsCount(*node, 2U, ctx)) {
+    if (!EnsureArgsCount(*node, 3U, ctx)) {
         return TStatus::Error;
     }
 
@@ -1177,6 +1177,10 @@ TStatus AnnotateOlapApply(const TExprNode::TPtr& node, TExprContext& ctx) {
         return TStatus::Repeat;
     }
 
+    if (!EnsureAtom(*node->Child(TKqpOlapApply::idx_KernelName), ctx)) {
+        return TStatus::Error;
+    }
+
     node->SetTypeAnn(lambda->GetTypeAnn());
     return TStatus::Ok;
 }
diff --git a/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter.cpp b/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter.cpp
@@ -253,6 +253,7 @@ TMaybeNode<TExprBase> YqlApplyPushdown(const TExprBase& apply, const TExprNode&
     return Build<TKqpOlapApply>(ctx, apply.Pos())
         .Lambda(ctx.NewLambda(apply.Pos(), ctx.NewArguments(argument.Pos(), std::move(lambdaArgs)), ctx.ReplaceNodes(apply.Ptr(), replacements)))
         .Args().Add(std::move(realArgs)).Build()
+        .KernelName(ctx.NewAtom(apply.Pos(), ""))    
         .Done();
 }
 
@@ -466,6 +467,32 @@ TExprBase BuildOneElementComparison(const std::pair<TExprBase, TExprBase>& param
             .Done();
     }
 
+    if (const auto* stringUdfFunction = IgnoreCaseSubstringMatchFunctions.FindPtr(predicate.CallableName())) {
+        const auto& leftArg = ctx.NewArgument(pos, "left");
+        const auto& rightArg = ctx.NewArgument(pos, "right");
+
+        const auto& callUdfLambda = ctx.NewLambda(pos, ctx.NewArguments(pos, {leftArg, rightArg}),
+            ctx.Builder(pos)
+                .Callable("Apply")
+                    .Callable(0, "Udf")
+                        .Atom(0, *stringUdfFunction)
+                    .Seal()
+                    .Add(1, leftArg)
+                    .Add(2, rightArg)
+                .Seal()
+            .Build()
+        );
+
+        return Build<TKqpOlapApply>(ctx, pos)
+            .Lambda(callUdfLambda)
+            .Args()
+                .Add(parameter.first)
+                .Add(parameter.second)
+            .Build()
+            .KernelName(ctx.NewAtom(pos, *stringUdfFunction))
+        .Done();
+    }
+
     std::string compareOperator = "";
 
     if (predicate.Maybe<TCoCmpEqual>()) {
diff --git a/ydb/core/kqp/opt/physical/predicate_collector.cpp b/ydb/core/kqp/opt/physical/predicate_collector.cpp
@@ -9,6 +9,13 @@ namespace NKikimr::NKqp::NOpt {
 using namespace NYql;
 using namespace NYql::NNodes;
 
+THashMap<TString, TString> IgnoreCaseSubstringMatchFunctions = {
+    {"EqualsIgnoreCase", "String.AsciiEqualsIgnoreCase"},
+    {"StartsWithIgnoreCase", "String.AsciiStartsWithIgnoreCase"},
+    {"EndsWithIgnoreCase", "String.AsciiEndsWithIgnoreCase"},
+    {"StringContainsIgnoreCase", "String.AsciiContainsIgnoreCase"}
+};
+
 namespace {
 
 bool IsSupportedPredicate(const TCoCompare& predicate) {
@@ -295,6 +302,18 @@ bool CheckComparisonParametersForPushdown(const TCoCompare& compare, const TExpr
         }
     }
 
+    if (options.PushdownSubstring) { //EnableSimpleIlikePushdown FF
+        if (IgnoreCaseSubstringMatchFunctions.contains(compare.CallableName())) {
+            const auto& right = compare.Right().Ref();
+            YQL_ENSURE(right.IsCallable("String") || right.IsCallable("Utf8"));
+            const auto pattern = right.Child(0);
+            YQL_ENSURE(pattern->IsAtom());
+            if (UTF8Detect(pattern->Content()) != ASCII) {
+                return false;
+            }
+        }
+    }
+
     return true;
 }
 
diff --git a/ydb/core/kqp/opt/physical/predicate_collector.h b/ydb/core/kqp/opt/physical/predicate_collector.h
@@ -21,6 +21,8 @@ struct TPushdownOptions {
     bool PushdownSubstring;
 };
 
+extern THashMap<TString, TString> IgnoreCaseSubstringMatchFunctions;
+
 void CollectPredicates(const NNodes::TExprBase& predicate, TOLAPPredicateNode& predicateTree, const TExprNode* lambdaArg, const NNodes::TExprBase& lambdaBody, const TPushdownOptions& options);
 
 }
diff --git a/ydb/core/kqp/query_compiler/kqp_olap_compiler.cpp b/ydb/core/kqp/query_compiler/kqp_olap_compiler.cpp
@@ -608,6 +608,7 @@ TTypedColumn CompileYqlKernelScalarApply(const TKqpOlapApply& apply, TKqpOlapCom
     const auto idx = ctx.GetKernelRequestBuilder().AddScalarApply(apply.Lambda().Ref(), argTypes, ctx.ExprCtx());
     function->SetKernelIdx(idx);
     function->SetFunctionType(TProgram::YQL_KERNEL);
+    function->SetKernelName(apply.KernelName().StringValue());
     std::for_each(ids.cbegin(), ids.cend(), [function] (ui64 id) { function->AddArguments()->SetId(id); });
     return {command->GetColumn().GetId(), ctx.ExprCtx().MakeType<TBlockExprType>(apply.Lambda().Body().Ref().GetTypeAnn())};
 }
diff --git a/ydb/core/kqp/ut/olap/combinatory/select.cpp b/ydb/core/kqp/ut/olap/combinatory/select.cpp
@@ -16,9 +16,10 @@ TConclusionStatus TSelectCommand::DoExecute(TKikimrRunner& kikimr) {
     const i64 headerApproveStart = controller->GetHeadersApprovedOnSelect().Val();
     const i64 headerNoDataStart = controller->GetHeadersSkippedNoData().Val();
 
-    Cerr << "EXECUTE: " << Command << Endl;
+    const auto command = "PRAGMA OptimizeSimpleILIKE; PRAGMA AnsiLIke;" + Command;
+    Cerr << "EXECUTE: " << command << Endl;
     auto session = kikimr.GetTableClient().CreateSession().GetValueSync().GetSession();
-    auto it = kikimr.GetQueryClient().StreamExecuteQuery(Command, NYdb::NQuery::TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+    auto it = kikimr.GetQueryClient().StreamExecuteQuery(command, NYdb::NQuery::TTxControl::BeginTx().CommitTx()).ExtractValueSync();
     UNIT_ASSERT_VALUES_EQUAL_C(it.GetStatus(), NYdb::EStatus::SUCCESS, it.GetIssues().ToString());
     TString output = StreamResultToYson(it);
     if (Compare) {
diff --git a/ydb/core/kqp/ut/olap/json_ut.cpp b/ydb/core/kqp/ut/olap/json_ut.cpp
@@ -633,22 +633,22 @@ Y_UNIT_TEST_SUITE(KqpOlapJson) {
             READ: SELECT * FROM `/Root/ColumnTable` WHERE JSON_VALUE(Col2, "$.\"b.c.d\"") like "%1b4%" ORDER BY Col1;
             EXPECTED: [[14u;["{\"a\":\"a4\",\"b.c.d\":\"1b4\"}"]]]
             IDX_ND_SKIP_APPROVE: 0, 4, 1
-#            ------
-#            READ: SELECT * FROM `/Root/ColumnTable` WHERE JSON_VALUE(Col2, "$.\"b.c.d\"") ilike "%1b4%" ORDER BY Col1;
-#            EXPECTED: [[14u;["{\"a\":\"a4\",\"b.c.d\":\"1b4\"}"]]]
-#            IDX_ND_SKIP_APPROVE: 0, 4, 1
-#            ------
-#            READ: SELECT * FROM `/Root/ColumnTable` WHERE JSON_VALUE(Col2, "$.\"b.c.d\"") ilike "%1B4" ORDER BY Col1;
-#            EXPECTED: [[14u;["{\"a\":\"a4\",\"b.c.d\":\"1b4\"}"]]]
-#            IDX_ND_SKIP_APPROVE: 0, 4, 1
-#            ------
-#            READ: SELECT * FROM `/Root/ColumnTable` WHERE JSON_VALUE(Col2, "$.\"b.c.d\"") ilike "1b5" ORDER BY Col1;
-#            EXPECTED: []
-#            IDX_ND_SKIP_APPROVE: 0, 5, 0
-#            ------
-#            READ: SELECT * FROM `/Root/ColumnTable` WHERE JSON_VALUE(Col2, "$.a") = "1b5" ORDER BY Col1;
-#            EXPECTED: []
-#            IDX_ND_SKIP_APPROVE: 0, 5, 0
+            ------
+            READ: SELECT * FROM `/Root/ColumnTable` WHERE JSON_VALUE(Col2, "$.\"b.c.d\"") ilike "%1b4%" ORDER BY Col1;
+            EXPECTED: [[14u;["{\"a\":\"a4\",\"b.c.d\":\"1b4\"}"]]]
+            IDX_ND_SKIP_APPROVE: 0, 4, 1
+            ------
+            READ: SELECT * FROM `/Root/ColumnTable` WHERE JSON_VALUE(Col2, "$.\"b.c.d\"") ilike "%1B4" ORDER BY Col1;
+            EXPECTED: [[14u;["{\"a\":\"a4\",\"b.c.d\":\"1b4\"}"]]]
+            IDX_ND_SKIP_APPROVE: 0, 4, 1
+            ------
+            READ: SELECT * FROM `/Root/ColumnTable` WHERE JSON_VALUE(Col2, "$.\"b.c.d\"") ilike "1b5" ORDER BY Col1;
+            EXPECTED: []
+            IDX_ND_SKIP_APPROVE: 0, 5, 0
+            ------
+            READ: SELECT * FROM `/Root/ColumnTable` WHERE JSON_VALUE(Col2, "$.a") = "1b5" ORDER BY Col1;
+            EXPECTED: []
+            IDX_ND_SKIP_APPROVE: 0, 5, 0
             ------
             READ: SELECT * FROM `/Root/ColumnTable` WHERE JSON_VALUE(Col2, "$.a") = "a4" ORDER BY Col1;
             EXPECTED: [[4u;["{\"a\":\"a4\",\"b.c.d\":\"b4\"}"]];[14u;["{\"a\":\"a4\",\"b.c.d\":\"1b4\"}"]]]
@@ -800,18 +800,18 @@ Y_UNIT_TEST_SUITE(KqpOlapJson) {
             READ: SELECT * FROM `/Root/ColumnTable` WHERE JSON_VALUE(Col2, "$.\"b.c.d\"") like "%1b4%" ORDER BY Col1;
             EXPECTED: [[14u;["{\"a\":\"a4\",\"b.c.d\":\"1b4\"}"]]]
             IDX_ND_SKIP_APPROVE: 0, 4, 1
-#            ------
-#            READ: SELECT * FROM `/Root/ColumnTable` WHERE JSON_VALUE(Col2, "$.\"b.c.d\"") ilike "%1b4%" ORDER BY Col1;
-#            EXPECTED: [[14u;["{\"a\":\"a4\",\"b.c.d\":\"1b4\"}"]]]
-#            IDX_ND_SKIP_APPROVE: 0, 4, 1
-#            ------
-#            READ: SELECT * FROM `/Root/ColumnTable` WHERE JSON_VALUE(Col2, "$.\"b.c.d\"") ilike "%1B4" ORDER BY Col1;
-#            EXPECTED: [[14u;["{\"a\":\"a4\",\"b.c.d\":\"1b4\"}"]]]
-#            IDX_ND_SKIP_APPROVE: 0, 4, 1
-#            ------
-#            READ: SELECT * FROM `/Root/ColumnTable` WHERE JSON_VALUE(Col2, "$.\"b.c.d\"") ilike "1b5" ORDER BY Col1;
-#            EXPECTED: []
-#            IDX_ND_SKIP_APPROVE: 0, 5, 0
+            ------
+            READ: SELECT * FROM `/Root/ColumnTable` WHERE JSON_VALUE(Col2, "$.\"b.c.d\"") ilike "%1b4%" ORDER BY Col1;
+            EXPECTED: [[14u;["{\"a\":\"a4\",\"b.c.d\":\"1b4\"}"]]]
+            IDX_ND_SKIP_APPROVE: 0, 4, 1
+            ------
+            READ: SELECT * FROM `/Root/ColumnTable` WHERE JSON_VALUE(Col2, "$.\"b.c.d\"") ilike "%1B4" ORDER BY Col1;
+            EXPECTED: [[14u;["{\"a\":\"a4\",\"b.c.d\":\"1b4\"}"]]]
+            IDX_ND_SKIP_APPROVE: 0, 4, 1
+            ------
+            READ: SELECT * FROM `/Root/ColumnTable` WHERE JSON_VALUE(Col2, "$.\"b.c.d\"") ilike "1b5" ORDER BY Col1;
+            EXPECTED: []
+            IDX_ND_SKIP_APPROVE: 0, 5, 0
             ------
             READ: SELECT * FROM `/Root/ColumnTable` WHERE JSON_VALUE(Col2, "$.a") = "1b5" ORDER BY Col1;
             EXPECTED: []
diff --git a/ydb/core/tx/program/builder.cpp b/ydb/core/tx/program/builder.cpp
@@ -20,8 +20,20 @@ namespace NKikimr::NArrow::NSSA {
 TConclusion<std::shared_ptr<IStepFunction>> TProgramBuilder::MakeFunction(const TColumnInfo& name,
     const NKikimrSSA::TProgram::TAssignment::TFunction& func, std::shared_ptr<NArrow::NSSA::IKernelLogic>& kernelLogic,
     std::vector<TColumnChainInfo>& arguments) const {
-    if (func.GetKernelName()) {
-        kernelLogic.reset(IKernelLogic::TFactory::Construct(func.GetKernelName()));
+    if (const auto& kernelName = func.GetKernelName(); !kernelName.empty()) {
+        if (kernelName == "String.AsciiEqualsIgnoreCase") {
+            kernelLogic = std::make_shared<TLogicMatchString>(TIndexCheckOperation::EOperation::Contains, false, false);
+        } else if (kernelName == "String.AsciiContainsIgnoreCase") {
+            kernelLogic = std::make_shared<TLogicMatchString>(TIndexCheckOperation::EOperation::Contains, false, false);
+        } else if (kernelName == "String.AsciiContainsIgnoreCase") {
+            kernelLogic = std::make_shared<TLogicMatchString>(TIndexCheckOperation::EOperation::Contains, false, false);
+        } else if (kernelName == "String.AsciiStartsWithIgnoreCase") {
+            kernelLogic = std::make_shared<TLogicMatchString>(TIndexCheckOperation::EOperation::StartsWith, false, false);
+        } else if (kernelName == "String.AsciiEndsWithIgnoreCase") {
+            kernelLogic = std::make_shared<TLogicMatchString>(TIndexCheckOperation::EOperation::EndsWith, false, false);
+        } else {
+            kernelLogic.reset(IKernelLogic::TFactory::Construct(kernelName));
+        }
     } else if (func.HasYqlOperationId()) {
         kernelLogic = std::make_shared<TSimpleKernelLogic>(func.GetYqlOperationId());
     } else {

Original file line number	Diff line number	Diff line change
`@@ -697,7 +697,8 @@`
`697`	`697`	`"Match": {"Type": "Callable", "Name": "KqpOlapApply"},`
`698`	`698`	`"Children": [`
`699`	`699`	`{"Index": 0, "Name": "Lambda", "Type": "TCoLambda"},`
`700`		`- {"Index": 1, "Name": "Args", "Type": "TExprList"}`
	`700`	`+ {"Index": 1, "Name": "Args", "Type": "TExprList"},`
	`701`	`+ {"Index": 2, "Name": "KernelName", "Type": "TCoAtom"}`
`701`	`702`	`]`
`702`	`703`	`},`
`703`	`704`	`{`
Original file line number	Diff line number	Diff line change
`@@ -1153,7 +1153,7 @@ TStatus AnnotateOlapApplyColumnArg(const TExprNode::TPtr& node, TExprContext& ct`
`1153`	`1153`	`}`
`1154`	`1154`
`1155`	`1155`	`TStatus AnnotateOlapApply(const TExprNode::TPtr& node, TExprContext& ctx) {`
`1156`		`- if (!EnsureArgsCount(*node, 2U, ctx)) {`
	`1156`	`+ if (!EnsureArgsCount(*node, 3U, ctx)) {`
`1157`	`1157`	`return TStatus::Error;`
`1158`	`1158`	`}`
`1159`	`1159`
`@@ -1177,6 +1177,10 @@ TStatus AnnotateOlapApply(const TExprNode::TPtr& node, TExprContext& ctx) {`
`1177`	`1177`	`return TStatus::Repeat;`
`1178`	`1178`	`}`
`1179`	`1179`
	`1180`	`+ if (!EnsureAtom(*node->Child(TKqpOlapApply::idx_KernelName), ctx)) {`
	`1181`	`+ return TStatus::Error;`
	`1182`	`+ }`
	`1183`	`+`
`1180`	`1184`	`node->SetTypeAnn(lambda->GetTypeAnn());`
`1181`	`1185`	`return TStatus::Ok;`
`1182`	`1186`	`}`
Original file line number	Diff line number	Diff line change
`@@ -21,6 +21,8 @@ struct TPushdownOptions {`
`21`	`21`	`bool PushdownSubstring;`
`22`	`22`	`};`
`23`	`23`
	`24`	`+extern THashMap<TString, TString> IgnoreCaseSubstringMatchFunctions;`
	`25`	`+`
`24`	`26`	`void CollectPredicates(const NNodes::TExprBase& predicate, TOLAPPredicateNode& predicateTree, const TExprNode* lambdaArg, const NNodes::TExprBase& lambdaBody, const TPushdownOptions& options);`
`25`	`27`
`26`	`28`	`}`
Original file line number	Diff line number	Diff line change
`@@ -608,6 +608,7 @@ TTypedColumn CompileYqlKernelScalarApply(const TKqpOlapApply& apply, TKqpOlapCom`
`608`	`608`	`const auto idx = ctx.GetKernelRequestBuilder().AddScalarApply(apply.Lambda().Ref(), argTypes, ctx.ExprCtx());`
`609`	`609`	`function->SetKernelIdx(idx);`
`610`	`610`	`function->SetFunctionType(TProgram::YQL_KERNEL);`
	`611`	`+ function->SetKernelName(apply.KernelName().StringValue());`
`611`	`612`	`std::for_each(ids.cbegin(), ids.cend(), [function] (ui64 id) { function->AddArguments()->SetId(id); });`
`612`	`613`	`return {command->GetColumn().GetId(), ctx.ExprCtx().MakeType<TBlockExprType>(apply.Lambda().Body().Ref().GetTypeAnn())};`
`613`	`614`	`}`