Skip to content

Commit 108c1aa

Browse files
vitstnvitstn
committed
YQL-19861 abi, udf resolvers, test, pass via yt gateways
commit_hash:6e3f5fac6a8598586987b52d749644d1ce1fccbe
1 parent 6b17ad8 commit 108c1aa

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+344
-141
lines changed

‎yql/essentials/core/type_ann/type_ann_core.cpp

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include <yql/essentials/minikql/dom/yson.h>
2323
#include <yql/essentials/utils/log/log.h>
2424
#include <yql/essentials/public/udf/udf_data_type.h>
25+
#include <yql/essentials/public/langver/yql_langver.h>
2526
#include <yql/essentials/providers/common/schema/expr/yql_expr_schema.h>
2627
#include <yql/essentials/utils/utf8.h>
2728

@@ -7606,14 +7607,39 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
76067607
if (!EnsureTupleSize(*child, 1, ctx.Expr)) {
76077608
return IGraphTransformer::TStatus::Error;
76087609
}
7609-
} else if (settingName == "cpu" || settingName == "extraMem") {
7610+
} else if (settingName == "cpu" || settingName == "extraMem" ||
7611+
settingName == "minLang" || settingName == "maxLang") {
76107612
if (!EnsureTupleSize(*child, 2, ctx.Expr)) {
76117613
return IGraphTransformer::TStatus::Error;
76127614
}
76137615

76147616
if (!EnsureAtom(child->Tail(), ctx.Expr)) {
76157617
return IGraphTransformer::TStatus::Error;
76167618
}
7619+
7620+
if (ctx.Types.LangVer != UnknownLangVersion) {
7621+
if (settingName == "minLang" && ctx.Types.LangVer < FromString<NYql::TLangVersion>(child->Tail().Content())) {
7622+
TLangVersionBuffer buffer;
7623+
TStringBuf str;
7624+
if (!FormatLangVersion(FromString<NYql::TLangVersion>(child->Tail().Content()), buffer, str)) {
7625+
str = "unknown";
7626+
}
7627+
7628+
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "UDF '" << name << "' is not available before version " << str));
7629+
return IGraphTransformer::TStatus::Error;
7630+
}
7631+
7632+
if (settingName == "maxLang" && ctx.Types.LangVer > FromString<NYql::TLangVersion>(child->Tail().Content())) {
7633+
TLangVersionBuffer buffer;
7634+
TStringBuf str;
7635+
if (!FormatLangVersion(FromString<NYql::TLangVersion>(child->Tail().Content()), buffer, str)) {
7636+
str = "unknown";
7637+
}
7638+
7639+
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "UDF '" << name << "' is not available after version " << str));
7640+
return IGraphTransformer::TStatus::Error;
7641+
}
7642+
}
76177643
} else {
76187644
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Head().Pos()), TStringBuilder() << "Unknown setting: " << settingName));
76197645
return IGraphTransformer::TStatus::Error;
@@ -7630,6 +7656,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
76307656
description.Name = TString(name);
76317657
description.UserType = userType;
76327658
description.TypeConfig = typeConfig;
7659+
description.LangVer = ctx.Types.LangVer;
76337660
ctx.Types.Credentials->ForEach([&description](const TString& name, const TCredential& cred) {
76347661
description.SecureParams[TString("token:") + name] = cred.Content;
76357662
if (name.StartsWith("default_")) {
@@ -7679,6 +7706,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
76797706
cached.NormalizedUserType = description.NormalizedUserType ? description.NormalizedUserType : ctx.Expr.MakeType<TVoidExprType>();
76807707
cached.SupportsBlocks = description.SupportsBlocks;
76817708
cached.IsStrict = description.IsStrict;
7709+
cached.MinLangVer = description.MinLangVer;
7710+
cached.MaxLangVer = description.MaxLangVer;
76827711

76837712
if (name != cached.NormalizedName) {
76847713
ctx.Types.UdfTypeCache[std::make_tuple(cached.NormalizedName, TString(typeConfig), userType)] = cached;
@@ -7733,6 +7762,20 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
77337762
.Seal();
77347763
}
77357764

7765+
if (cached.MinLangVer != UnknownLangVersion) {
7766+
parent.List(settingIndex++)
7767+
.Atom(0, "minLang")
7768+
.Atom(1, cached.MinLangVer)
7769+
.Seal();
7770+
}
7771+
7772+
if (cached.MaxLangVer != UnknownLangVersion) {
7773+
parent.List(settingIndex++)
7774+
.Atom(0, "maxLang")
7775+
.Atom(1, cached.MaxLangVer)
7776+
.Seal();
7777+
}
7778+
77367779
if (settings) {
77377780
if (auto setting = GetSetting(*settings, "cpu")) {
77387781
parent.Add(settingIndex++, setting);

‎yql/essentials/core/type_ann/ya.make

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ PEERDIR(
3636
yql/essentials/parser/pg_catalog
3737
yql/essentials/core/sql_types
3838
yql/essentials/parser/pg_wrapper/interface
39+
yql/essentials/public/langver
3940
library/cpp/yson/node
4041
)
4142

‎yql/essentials/core/yql_type_annotation.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -339,6 +339,8 @@ struct TUdfCachedInfo {
339339
const TTypeAnnotationNode* NormalizedUserType = nullptr;
340340
bool SupportsBlocks = false;
341341
bool IsStrict = false;
342+
TLangVersion MinLangVer = UnknownLangVersion;
343+
TLangVersion MaxLangVer = UnknownLangVersion;
342344
};
343345

344346
const TString TypeAnnotationContextComponent = "TypeAnnotationContext";

‎yql/essentials/core/yql_udf_resolver.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include <yql/essentials/providers/common/proto/udf_resolver.pb.h>
55

66
#include <yql/essentials/core/yql_holding_file_storage.h>
7+
#include <yql/essentials/public/langver/yql_langver.h>
78
#include <yql/essentials/public/issue/yql_issue.h>
89
#include <yql/essentials/public/udf/udf_log.h>
910

@@ -42,6 +43,7 @@ class IUdfResolver : public TThrRefBase {
4243
TString TypeConfig;
4344
const TTypeAnnotationNode* UserType = nullptr;
4445
THashMap<TString, TString> SecureParams;
46+
NYql::TLangVersion LangVer = NYql::UnknownLangVersion;
4547

4648
// output
4749
TString NormalizedName;
@@ -50,6 +52,8 @@ class IUdfResolver : public TThrRefBase {
5052
const TTypeAnnotationNode* CallableType = nullptr;
5153
bool SupportsBlocks = false;
5254
bool IsStrict = false;
55+
NYql::TLangVersion MinLangVer = NYql::UnknownLangVersion;
56+
NYql::TLangVersion MaxLangVer = NYql::UnknownLangVersion;
5357
TVector<TString> Messages;
5458
};
5559

‎yql/essentials/minikql/comp_nodes/mkql_scalar_apply.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ class TScalarApplyWrapper : public TMutableComputationNode<TScalarApplyWrapper>
5555
, TimeProvider(CreateDefaultTimeProvider())
5656
, Ctx(HolderFactory, &ValueBuilder, TComputationOptsFull(
5757
nullptr, Alloc.Ref(), TypeEnv, *RandomProvider, *TimeProvider, NUdf::EValidatePolicy::Exception, originalContext.SecureParamsProvider,
58-
originalContext.CountersProvider, originalContext.LogProvider),
58+
originalContext.CountersProvider, originalContext.LogProvider, originalContext.LangVer),
5959
originalContext.Mutables, *NYql::NUdf::GetYqlMemoryPool())
6060
{
6161
Alloc.Ref().EnableArrowTracking = false;

‎yql/essentials/minikql/comp_nodes/mkql_udf.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ using TBaseComputation = TMutableComputationNode<TSimpleUdfWrapper<TValidatePoli
5050
ui32 flags = 0;
5151
TFunctionTypeInfo funcInfo;
5252
const auto status = ctx.HolderFactory.GetFunctionRegistry()->FindFunctionTypeInfo(
53-
ctx.TypeEnv, ctx.TypeInfoHelper, ctx.CountersProvider, FunctionName, UserType->IsVoid() ? nullptr : UserType,
53+
ctx.LangVer, ctx.TypeEnv, ctx.TypeInfoHelper, ctx.CountersProvider, FunctionName, UserType->IsVoid() ? nullptr : UserType,
5454
TypeConfig, flags, Pos, ctx.SecureParamsProvider, ctx.LogProvider, &funcInfo);
5555

5656
if (!status.IsOk()) {
@@ -204,7 +204,7 @@ using TBaseComputation = TMutableCodegeneratorPtrNode<TUdfWrapper<TValidatePolic
204204
ui32 flags = 0;
205205
TFunctionTypeInfo funcInfo;
206206
const auto status = ctx.HolderFactory.GetFunctionRegistry()->FindFunctionTypeInfo(
207-
ctx.TypeEnv, ctx.TypeInfoHelper, ctx.CountersProvider, FunctionName, UserType->IsVoid() ? nullptr : UserType,
207+
ctx.LangVer, ctx.TypeEnv, ctx.TypeInfoHelper, ctx.CountersProvider, FunctionName, UserType->IsVoid() ? nullptr : UserType,
208208
TypeConfig, flags, Pos, ctx.SecureParamsProvider, ctx.LogProvider, &funcInfo);
209209

210210
if (!status.IsOk()) {
@@ -290,7 +290,7 @@ IComputationNode* WrapUdf(TCallable& callable, const TComputationNodeFactoryCont
290290
const auto userType = static_cast<TType*>(userTypeNode.GetNode());
291291

292292
const auto status = ctx.FunctionRegistry.FindFunctionTypeInfo(
293-
ctx.Env, ctx.TypeInfoHelper, ctx.CountersProvider, funcName, userType->IsVoid() ? nullptr : userType,
293+
ctx.LangVer, ctx.Env, ctx.TypeInfoHelper, ctx.CountersProvider, funcName, userType->IsVoid() ? nullptr : userType,
294294
typeConfig, flags, pos, ctx.SecureParamsProvider, ctx.LogProvider, &funcInfo);
295295

296296
if (!status.IsOk()) {
@@ -352,7 +352,7 @@ IComputationNode* WrapScriptUdf(TCallable& callable, const TComputationNodeFacto
352352
ui32 flags = 0;
353353
TFunctionTypeInfo funcInfo;
354354
const auto status = ctx.FunctionRegistry.FindFunctionTypeInfo(
355-
ctx.Env, ctx.TypeInfoHelper, ctx.CountersProvider, funcName, userType,
355+
ctx.LangVer, ctx.Env, ctx.TypeInfoHelper, ctx.CountersProvider, funcName, userType,
356356
typeConfig, flags, pos, ctx.SecureParamsProvider, ctx.LogProvider, &funcInfo);
357357

358358
if (!status.IsOk()) {

‎yql/essentials/minikql/computation/mkql_computation_node.h

Lines changed: 32 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include <yql/essentials/minikql/mkql_stats_registry.h>
1212
#include <yql/essentials/minikql/mkql_terminator.h>
1313

14+
#include <yql/essentials/public/langver/yql_langver.h>
1415
#include <yql/essentials/public/udf/udf_value.h>
1516
#include <yql/essentials/public/udf/udf_validate.h>
1617
#include <yql/essentials/public/udf/udf_value_builder.h>
@@ -46,7 +47,7 @@ struct TComputationOpts {
4647
struct TComputationOptsFull: public TComputationOpts {
4748
TComputationOptsFull(IStatsRegistry* stats, TAllocState& allocState, const TTypeEnvironment& typeEnv, IRandomProvider& randomProvider,
4849
ITimeProvider& timeProvider, NUdf::EValidatePolicy validatePolicy, const NUdf::ISecureParamsProvider* secureParamsProvider,
49-
NUdf::ICountersProvider* countersProvider, const NUdf::ILogProvider* logProvider)
50+
NUdf::ICountersProvider* countersProvider, const NUdf::ILogProvider* logProvider, NYql::TLangVersion langver)
5051
: TComputationOpts(stats)
5152
, AllocState(allocState)
5253
, TypeEnv(typeEnv)
@@ -56,6 +57,7 @@ struct TComputationOptsFull: public TComputationOpts {
5657
, SecureParamsProvider(secureParamsProvider)
5758
, CountersProvider(countersProvider)
5859
, LogProvider(logProvider)
60+
, LangVer(langver)
5961
{}
6062

6163
TAllocState& AllocState;
@@ -66,6 +68,7 @@ struct TComputationOptsFull: public TComputationOpts {
6668
const NUdf::ISecureParamsProvider *const SecureParamsProvider;
6769
NUdf::ICountersProvider *const CountersProvider;
6870
const NUdf::ILogProvider* const LogProvider;
71+
const NYql::TLangVersion LangVer;
6972
};
7073

7174
struct TWideFieldsInitInfo {
@@ -121,6 +124,7 @@ struct TComputationContext : public TComputationContextLLVM {
121124
NUdf::ICountersProvider *const CountersProvider;
122125
const NUdf::ISecureParamsProvider *const SecureParamsProvider;
123126
const NUdf::ILogProvider* LogProvider;
127+
NYql::TLangVersion LangVer = NYql::UnknownLangVersion;
124128

125129
TComputationContext(const THolderFactory& holderFactory,
126130
const NUdf::IValueBuilder* builder,
@@ -280,6 +284,7 @@ struct TComputationNodeFactoryContext {
280284
NUdf::ICountersProvider* CountersProvider;
281285
const NUdf::ISecureParamsProvider* SecureParamsProvider;
282286
const NUdf::ILogProvider* LogProvider;
287+
NYql::TLangVersion LangVer;
283288
const TNodeFactory& NodeFactory;
284289
const THolderFactory& HolderFactory;
285290
const NUdf::IValueBuilder *const Builder;
@@ -291,30 +296,32 @@ struct TComputationNodeFactoryContext {
291296
const TNodePushBack NodePushBack;
292297

293298
TComputationNodeFactoryContext(
294-
const TNodeLocator& nodeLocator,
295-
const IFunctionRegistry& functionRegistry,
296-
const TTypeEnvironment& env,
297-
NUdf::ITypeInfoHelper::TPtr typeInfoHelper,
298-
NUdf::ICountersProvider* countersProvider,
299-
const NUdf::ISecureParamsProvider* secureParamsProvider,
300-
const NUdf::ILogProvider* logProvider,
301-
const TNodeFactory& nodeFactory,
302-
const THolderFactory& holderFactory,
303-
const NUdf::IValueBuilder* builder,
304-
NUdf::EValidateMode validateMode,
305-
NUdf::EValidatePolicy validatePolicy,
306-
EGraphPerProcess graphPerProcess,
307-
TComputationMutables& mutables,
308-
TComputationNodeOnNodeMap& elementsCache,
309-
TNodePushBack&& nodePushBack
310-
)
299+
const TNodeLocator& nodeLocator,
300+
const IFunctionRegistry& functionRegistry,
301+
const TTypeEnvironment& env,
302+
NUdf::ITypeInfoHelper::TPtr typeInfoHelper,
303+
NUdf::ICountersProvider* countersProvider,
304+
const NUdf::ISecureParamsProvider* secureParamsProvider,
305+
const NUdf::ILogProvider* logProvider,
306+
NYql::TLangVersion langver,
307+
const TNodeFactory& nodeFactory,
308+
const THolderFactory& holderFactory,
309+
const NUdf::IValueBuilder* builder,
310+
NUdf::EValidateMode validateMode,
311+
NUdf::EValidatePolicy validatePolicy,
312+
EGraphPerProcess graphPerProcess,
313+
TComputationMutables& mutables,
314+
TComputationNodeOnNodeMap& elementsCache,
315+
TNodePushBack&& nodePushBack
316+
)
311317
: NodeLocator(nodeLocator)
312318
, FunctionRegistry(functionRegistry)
313319
, Env(env)
314320
, TypeInfoHelper(typeInfoHelper)
315321
, CountersProvider(countersProvider)
316322
, SecureParamsProvider(secureParamsProvider)
317323
, LogProvider(logProvider)
324+
, LangVer(langver)
318325
, NodeFactory(nodeFactory)
319326
, HolderFactory(holderFactory)
320327
, Builder(builder)
@@ -350,7 +357,8 @@ struct TComputationPatternOpts {
350357
IStatsRegistry* stats = nullptr,
351358
NUdf::ICountersProvider* countersProvider = nullptr,
352359
const NUdf::ISecureParamsProvider* secureParamsProvider = nullptr,
353-
const NUdf::ILogProvider* logProvider = nullptr)
360+
const NUdf::ILogProvider* logProvider = nullptr,
361+
NYql::TLangVersion langver = NYql::UnknownLangVersion)
354362
: AllocState(allocState)
355363
, Env(env)
356364
, Factory(factory)
@@ -363,14 +371,15 @@ struct TComputationPatternOpts {
363371
, CountersProvider(countersProvider)
364372
, SecureParamsProvider(secureParamsProvider)
365373
, LogProvider(logProvider)
374+
, LangVer(langver)
366375
{}
367376

368377
void SetOptions(TComputationNodeFactory factory, const IFunctionRegistry* functionRegistry,
369378
NUdf::EValidateMode validateMode, NUdf::EValidatePolicy validatePolicy,
370379
const TString& optLLVM, EGraphPerProcess graphPerProcess, IStatsRegistry* stats = nullptr,
371380
NUdf::ICountersProvider* counters = nullptr,
372381
const NUdf::ISecureParamsProvider* secureParamsProvider = nullptr,
373-
const NUdf::ILogProvider* logProvider = nullptr) {
382+
const NUdf::ILogProvider* logProvider = nullptr, NYql::TLangVersion langver = NYql::UnknownLangVersion) {
374383
Factory = factory;
375384
FunctionRegistry = functionRegistry;
376385
ValidateMode = validateMode;
@@ -381,6 +390,7 @@ struct TComputationPatternOpts {
381390
CountersProvider = counters;
382391
SecureParamsProvider = secureParamsProvider;
383392
LogProvider = logProvider;
393+
LangVer = langver;
384394
}
385395

386396
void SetPatternEnv(std::shared_ptr<TPatternCacheEntry> cacheEnv) {
@@ -401,10 +411,11 @@ struct TComputationPatternOpts {
401411
NUdf::ICountersProvider* CountersProvider = nullptr;
402412
const NUdf::ISecureParamsProvider* SecureParamsProvider = nullptr;
403413
const NUdf::ILogProvider* LogProvider = nullptr;
414+
NYql::TLangVersion LangVer = NYql::UnknownLangVersion;
404415

405416
TComputationOptsFull ToComputationOptions(IRandomProvider& randomProvider, ITimeProvider& timeProvider, TAllocState* allocStatePtr = nullptr) const {
406417
return TComputationOptsFull(Stats, allocStatePtr ? *allocStatePtr : AllocState, Env, randomProvider, timeProvider,
407-
ValidatePolicy, SecureParamsProvider, CountersProvider, LogProvider);
418+
ValidatePolicy, SecureParamsProvider, CountersProvider, LogProvider, LangVer);
408419
}
409420
};
410421

‎yql/essentials/minikql/computation/mkql_computation_node_graph.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,7 @@ class TComputationGraphBuildingVisitor:
239239
, CountersProvider(opts.CountersProvider)
240240
, SecureParamsProvider(opts.SecureParamsProvider)
241241
, LogProvider(opts.LogProvider)
242+
, LangVer(opts.LangVer)
242243
, Factory(opts.Factory)
243244
, FunctionRegistry(*opts.FunctionRegistry)
244245
, ValidateMode(opts.ValidateMode)
@@ -465,6 +466,7 @@ class TComputationGraphBuildingVisitor:
465466
CountersProvider,
466467
SecureParamsProvider,
467468
LogProvider,
469+
LangVer,
468470
*NodeFactory,
469471
*PatternNodes->HolderFactory,
470472
PatternNodes->ValueBuilder.Get(),
@@ -562,6 +564,7 @@ class TComputationGraphBuildingVisitor:
562564
NUdf::ICountersProvider* CountersProvider;
563565
const NUdf::ISecureParamsProvider* SecureParamsProvider;
564566
const NUdf::ILogProvider* LogProvider;
567+
const NYql::TLangVersion LangVer;
565568
const TComputationNodeFactory Factory;
566569
const IFunctionRegistry& FunctionRegistry;
567570
TIntrusivePtr<TMemoryUsageInfo> MemInfo;

‎yql/essentials/minikql/computation/mkql_validate_ut.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -683,7 +683,8 @@ Y_UNIT_TEST_SUITE(TMiniKQLValidateTest) {
683683
TFunctionTypeInfo funcInfo;
684684
TType* userType = nullptr;
685685
TStringBuf typeConfig;
686-
TStatus status = functionRegistry->FindFunctionTypeInfo(env, typeInfoHelper, nullptr, udfFuncName, userType, typeConfig, flags, {}, nullptr, nullptr, &funcInfo);
686+
TStatus status = functionRegistry->FindFunctionTypeInfo(NYql::UnknownLangVersion, env, typeInfoHelper,
687+
nullptr, udfFuncName, userType, typeConfig, flags, {}, nullptr, nullptr, &funcInfo);
687688
MKQL_ENSURE(status.IsOk(), status.GetError());
688689
auto type = funcInfo.FunctionType->GetReturnType();
689690
fullValidateFunc(value, builder, type);
@@ -733,7 +734,8 @@ Y_UNIT_TEST_SUITE(TMiniKQLValidateTest) {
733734
TType* userType = nullptr;
734735
TStringBuf typeConfig;
735736
NUdf::ITypeInfoHelper::TPtr typeInfoHelper(new TTypeInfoHelper);
736-
TStatus status = functionRegistry.FindFunctionTypeInfo(pgmBuilder.GetTypeEnvironment(), typeInfoHelper, nullptr,
737+
TStatus status = functionRegistry.FindFunctionTypeInfo(NYql::UnknownLangVersion,
738+
pgmBuilder.GetTypeEnvironment(), typeInfoHelper, nullptr,
737739
udfFuncName, userType, typeConfig, flags, {}, nullptr, nullptr, &funcInfo);
738740
MKQL_ENSURE(status.IsOk(), status.GetError());
739741
auto callable = pgmBuilder.Udf(udfFuncName);

‎yql/essentials/minikql/computation/mkql_value_builder_ut.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ class TMiniKQLValueBuilderTest: public TTestBase {
4444
, HolderFactory(Alloc.Ref(), MemInfo, FunctionRegistry.Get())
4545
, Builder(HolderFactory, NUdf::EValidatePolicy::Exception)
4646
, TypeInfoHelper(new TTypeInfoHelper())
47-
, FunctionTypeInfoBuilder(Env, TypeInfoHelper, "", nullptr, {})
47+
, FunctionTypeInfoBuilder(NYql::UnknownLangVersion, Env, TypeInfoHelper, "", nullptr, {})
4848
{
4949
BoolOid = NYql::NPg::LookupType("bool").TypeId;
5050
}

0 commit comments

Comments
 (0)